feat(0077): prefer a robust parameter plateau over the in-sample peak
Add an opt-in plateau selection objective for walk-forward, recorded on the same RunManifest.selection carrier #144 introduced. Default argmax is byte-identical (C23); plateau is reached only via a new --select flag. What landed: - FamilySelection / SelectionMode reshape (report.rs): the selection RULE (mode) and its ANNOTATION are orthogonal — deflation fields become Option (present iff Argmax), plus PlateauMean/PlateauWorst variants and neighbourhood_score/n_neighbours (present iff Plateau*). Legacy lines still load: the deflation scalars deserialize from bare values via serde default (C14/C18). compat.rs embeds FamilySelection by value — reshape flows through untouched. - GridSpace::axis_lens() + SweepBinder::sweep_with_lattice (engine): the grid radixes in param_space()/odometer order. sweep() delegates to sweep_with_lattice with the lattice dropped, so every existing .sweep() caller is byte-unchanged. - optimize_plateau + PlateauMode + closed_neighbourhood (aura-registry, C9): each member scores as the mean/worst of its closed mixed-radix grid neighbourhood's metric_value; the winner is the smoothed argmax by the metric's direction (earliest-odometer tie, as optimize). Pure, no RNG (C1); in-sample only (C2). A higher_is_better helper now sources the per-metric direction once, shared by metric_cmp and optimize_plateau. - CLI --select <argmax|plateau:mean|plateau:worst> (default argmax; unknown token exits 2). walkforward_family dispatches via a select_winner helper; sweep_over/stage1_r_sweep_over carry the lattice as Option<Vec<usize>>. runs-family display gains a plateau(<mode>)=<score> over <n> cells line. RandomSpace-refuse: walkforward has no --random producer, so the plateau-without-lattice guard is structural — select_winner returns the refuse on a None lattice (the caller prints the message and exits 2), unit-tested at the helper, not via a --random E2E (decided + recorded on #145 comment 1974, with the lattice-seam fork). Tests: plateau-picks-plateau-not-spike, mean-vs-worst, lower-is-better direction-flip, closed-neighbourhood golden, single-member degeneracy, C1 determinism; CLI select-parse, select_winner refuse; E2E argmax-byte-identical (C23), plateau:mean and plateau:worst provenance. Full workspace suite green; clippy --all-targets -D warnings clean. closes #145
This commit is contained in:
+138
-40
@@ -18,15 +18,15 @@ use aura_core::{zip_params, Cell, Firing, ParamSpec, Scalar, ScalarKind, Timesta
|
||||
use aura_composites::{risk_executor, risk_executor_vol_open, StopRule};
|
||||
use aura_engine::{
|
||||
f64_field, join_on_ts, monte_carlo, param_stability, r_bootstrap, r_metrics_from_rs, summarize,
|
||||
summarize_r, walk_forward, window_of, ColumnarTrace, Composite, Edge, FlatGraph, GraphBuilder,
|
||||
Harness, JoinedRow, McAggregate, McFamily, RBootstrap, RollMode, RunManifest, RunMetrics,
|
||||
RunReport, SourceSpec, SweepFamily, SweepPoint, SyntheticSpec, Target, VecSource,
|
||||
WalkForwardResult, WindowBounds, WindowRoller, WindowRun,
|
||||
summarize_r, walk_forward, window_of, ColumnarTrace, Composite, Edge, FamilySelection, FlatGraph,
|
||||
GraphBuilder, Harness, JoinedRow, McAggregate, McFamily, RBootstrap, RollMode, RunManifest,
|
||||
RunMetrics, RunReport, SelectionMode, SourceSpec, SweepFamily, SweepPoint, SyntheticSpec, Target,
|
||||
VecSource, WalkForwardResult, WindowBounds, WindowRoller, WindowRun,
|
||||
};
|
||||
use aura_registry::{
|
||||
group_families, mc_member_reports, optimize_deflated, rank_by, sweep_member_reports,
|
||||
walkforward_member_reports, FamilyKind, FamilyMember, NameKind, Registry, RunTraces, TraceStore,
|
||||
WriteKind,
|
||||
group_families, mc_member_reports, optimize_deflated, optimize_plateau, rank_by,
|
||||
sweep_member_reports, walkforward_member_reports, FamilyKind, FamilyMember, NameKind,
|
||||
PlateauMode, Registry, RunTraces, TraceStore, WriteKind,
|
||||
};
|
||||
use aura_std::{
|
||||
Add, Bias, Delay, Ema, GatedRecorder, Gt, Latch, LinComb, LongOnly, Mul, Recorder, RollingMax,
|
||||
@@ -1298,7 +1298,7 @@ fn stage1_r_space() -> Vec<ParamSpec> {
|
||||
/// but windowed (`windowed_sources`) and always folded (O(trades)/member): each
|
||||
/// member's RunReport carries `metrics.r = Some(summarize_r(..))`, so the family is
|
||||
/// rankable by an R metric.
|
||||
fn stage1_r_sweep_over(from: Timestamp, to: Timestamp, data: &DataSource, grid: &Stage1RGrid) -> SweepFamily {
|
||||
fn stage1_r_sweep_over(from: Timestamp, to: Timestamp, data: &DataSource, grid: &Stage1RGrid) -> (SweepFamily, Option<Vec<usize>>) {
|
||||
let pip = data.pip_size();
|
||||
let (tx_eq, _) = mpsc::channel();
|
||||
let (tx_ex, _) = mpsc::channel();
|
||||
@@ -1315,7 +1315,7 @@ fn stage1_r_sweep_over(from: Timestamp, to: Timestamp, data: &DataSource, grid:
|
||||
.axis("slow.length", grid.slow.clone())
|
||||
.axis(&stop_length_axis, grid.stop_length.clone())
|
||||
.axis(&stop_k_axis, grid.stop_k.clone())
|
||||
.sweep(|point| {
|
||||
.sweep_with_lattice(|point| {
|
||||
let (tx_eq, rx_eq) = mpsc::channel();
|
||||
let (tx_ex, rx_ex) = mpsc::channel();
|
||||
let (tx_r, rx_r) = mpsc::channel();
|
||||
@@ -1339,6 +1339,7 @@ fn stage1_r_sweep_over(from: Timestamp, to: Timestamp, data: &DataSource, grid:
|
||||
m.r = Some(summarize_r(&r_rows, 0.0));
|
||||
RunReport { manifest, metrics: m }
|
||||
})
|
||||
.map(|(fam, lat)| (fam, Some(lat)))
|
||||
.expect("the stage1-r named grid matches the stage1-r param-space")
|
||||
}
|
||||
|
||||
@@ -1556,6 +1557,26 @@ enum Strategy {
|
||||
Stage1MeanRev,
|
||||
}
|
||||
|
||||
/// In-sample winner-selection objective for walk-forward (cycle 0077). `Argmax` is
|
||||
/// the bare-best pick deflated for trials (#144, the default); `Plateau` argmaxes
|
||||
/// the neighbourhood-smoothed surface instead (opt-in via `--select`).
|
||||
#[derive(Clone, Copy)]
|
||||
enum Selection {
|
||||
Argmax,
|
||||
Plateau(PlateauMode),
|
||||
}
|
||||
|
||||
/// Parse a `--select` token: `argmax` | `plateau:mean` | `plateau:worst`. Unknown
|
||||
/// tokens are a usage error (the caller maps `Err(())` to exit 2).
|
||||
fn parse_select(s: &str) -> Result<Selection, ()> {
|
||||
match s {
|
||||
"argmax" => Ok(Selection::Argmax),
|
||||
"plateau:mean" => Ok(Selection::Plateau(PlateauMode::Mean)),
|
||||
"plateau:worst" => Ok(Selection::Plateau(PlateauMode::Worst)),
|
||||
_ => Err(()),
|
||||
}
|
||||
}
|
||||
|
||||
impl Strategy {
|
||||
/// The CLI `--strategy` token this variant parses from — the inverse of the
|
||||
/// `parse_*_args` match arms. Used to echo the offending strategy in error
|
||||
@@ -1642,17 +1663,18 @@ fn parse_sweep_args(
|
||||
}
|
||||
|
||||
/// Parse the `walkforward` tail:
|
||||
/// `[--strategy <sma|momentum|stage1-r|stage1-breakout|stage1-meanrev>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--name <n> | --trace <n>] [--fast <csv>] [--slow <csv>] [--stop-length <csv>] [--stop-k <csv>]`.
|
||||
/// `[--strategy <sma|momentum|stage1-r|stage1-breakout|stage1-meanrev>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--name <n> | --trace <n>] [--fast <csv>] [--slow <csv>] [--stop-length <csv>] [--stop-k <csv>] [--select <argmax|plateau:mean|plateau:worst>]`.
|
||||
/// Defaults: SMA-cross, synthetic, name "walkforward", no persist. `--name`/`--trace`
|
||||
/// mutually exclusive; `--from`/`--to` require `--real`. Pure (no I/O / exit).
|
||||
fn parse_walkforward_args(
|
||||
rest: &[&str],
|
||||
) -> Result<(Strategy, String, bool, DataChoice, Stage1RGrid), String> {
|
||||
let usage = || "walkforward [--strategy <sma|momentum|stage1-r|stage1-breakout|stage1-meanrev>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--name <n> | --trace <n>] [--fast <csv>] [--slow <csv>] [--stop-length <csv>] [--stop-k <csv>]".to_string();
|
||||
) -> Result<(Strategy, String, bool, DataChoice, Stage1RGrid, Selection), String> {
|
||||
let usage = || "walkforward [--strategy <sma|momentum|stage1-r|stage1-breakout|stage1-meanrev>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--name <n> | --trace <n>] [--fast <csv>] [--slow <csv>] [--stop-length <csv>] [--stop-k <csv>] [--select <argmax|plateau:mean|plateau:worst>]".to_string();
|
||||
let mut strategy = Strategy::SmaCross;
|
||||
let mut name: Option<(String, bool)> = None;
|
||||
let mut real = RealWindowGrammar::default();
|
||||
let mut grid = Stage1RGrid::default();
|
||||
let mut select = Selection::Argmax;
|
||||
let mut tail = rest;
|
||||
while let Some((flag, t)) = tail.split_first() {
|
||||
let (value, t) = t.split_first().ok_or_else(usage)?;
|
||||
@@ -1677,12 +1699,13 @@ fn parse_walkforward_args(
|
||||
"--slow" => grid.slow = parse_csv_list(value).map_err(|()| usage())?,
|
||||
"--stop-length" => grid.stop_length = parse_csv_list(value).map_err(|()| usage())?,
|
||||
"--stop-k" => grid.stop_k = parse_csv_list(value).map_err(|()| usage())?,
|
||||
"--select" => select = parse_select(value).map_err(|()| usage())?,
|
||||
_ => return Err(usage()),
|
||||
}
|
||||
tail = t;
|
||||
}
|
||||
let (name, persist) = name.unwrap_or_else(|| ("walkforward".to_string(), false));
|
||||
Ok((strategy, name, persist, real.finish(&usage)?, grid))
|
||||
Ok((strategy, name, persist, real.finish(&usage)?, grid, select))
|
||||
}
|
||||
|
||||
/// Render a family-member stdout line: the assigned `family_id` plus the embedded
|
||||
@@ -1752,7 +1775,7 @@ fn run_sweep(strategy: Strategy, name: &str, persist: bool, data: DataSource, gr
|
||||
/// print each carrying the assigned id, then the stitched summary line. With
|
||||
/// `--trace`, also persist each OOS member's streams under
|
||||
/// `runs/traces/<n>/oos<ns>/` (opt-in). Deterministic (C1).
|
||||
fn run_walkforward(strategy: Strategy, name: &str, persist: bool, data: DataSource, grid: &Stage1RGrid) {
|
||||
fn run_walkforward(strategy: Strategy, name: &str, persist: bool, data: DataSource, grid: &Stage1RGrid, select: Selection) {
|
||||
if persist
|
||||
&& let Err(e) = TraceStore::open("runs").ensure_name_free(name, WriteKind::Family)
|
||||
{
|
||||
@@ -1760,7 +1783,7 @@ fn run_walkforward(strategy: Strategy, name: &str, persist: bool, data: DataSour
|
||||
std::process::exit(2);
|
||||
}
|
||||
let reg = default_registry();
|
||||
let result = walkforward_family(strategy, persist.then_some(name), &data, grid);
|
||||
let result = walkforward_family(strategy, persist.then_some(name), &data, grid, select);
|
||||
let id =
|
||||
match reg.append_family(name, FamilyKind::WalkForward, &walkforward_member_reports(&result))
|
||||
{
|
||||
@@ -1776,6 +1799,31 @@ fn run_walkforward(strategy: Strategy, name: &str, persist: bool, data: DataSour
|
||||
println!("{}", walkforward_summary_json(&result));
|
||||
}
|
||||
|
||||
/// Resolve the in-sample winner under the chosen selection objective. `Argmax`
|
||||
/// defers to the trials-deflation pick (#144). `Plateau` argmaxes the smoothed grid
|
||||
/// surface — it needs the grid lattice, so a sweep with no lattice (a future random
|
||||
/// walk-forward producer) is refused rather than silently argmaxed. The metric is
|
||||
/// always known at the call sites, so a metric error is unreachable (`expect`); the
|
||||
/// only fallible outcome is the plateau-without-lattice refusal, returned as
|
||||
/// `Err(message)` for the caller to print and exit 2.
|
||||
fn select_winner(
|
||||
family: &SweepFamily, metric: &str, select: Selection, lattice: Option<&[usize]>,
|
||||
) -> Result<(SweepPoint, FamilySelection), String> {
|
||||
match select {
|
||||
Selection::Argmax => Ok(optimize_deflated(
|
||||
family, metric, DEFLATION_N_RESAMPLES, DEFLATION_BLOCK_LEN, DEFLATION_SEED,
|
||||
).expect("walk-forward metrics are known")),
|
||||
Selection::Plateau(mode) => match lattice {
|
||||
Some(lens) => Ok(optimize_plateau(family, lens, metric, mode)
|
||||
.expect("walk-forward metrics are known")),
|
||||
None => Err(
|
||||
"--select plateau requires a grid sweep; a random sweep has no parameter lattice"
|
||||
.to_string(),
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// The built-in rolling walk-forward: 24-bar in-sample, 12-bar out-of-sample,
|
||||
/// stepping 12 (contiguous OOS tiling), over the 60-bar synthetic span -> 3
|
||||
/// windows. Each window sweeps a grid in-sample, optimizes by a metric, and runs
|
||||
@@ -1785,6 +1833,7 @@ fn run_walkforward(strategy: Strategy, name: &str, persist: bool, data: DataSour
|
||||
/// Other strategies have no walk-forward form yet (exit 2).
|
||||
fn walkforward_family(
|
||||
strategy: Strategy, trace: Option<&str>, data: &DataSource, grid: &Stage1RGrid,
|
||||
select: Selection,
|
||||
) -> WalkForwardResult {
|
||||
let span = data.wf_full_span();
|
||||
let (is_len, oos_len, step) = data.wf_window_sizes();
|
||||
@@ -1799,10 +1848,11 @@ fn walkforward_family(
|
||||
Strategy::SmaCross => {
|
||||
let space = sample_blueprint_with_sinks(data.pip_size()).0.param_space();
|
||||
walk_forward(roller, space, |w: WindowBounds| {
|
||||
let is_family = sweep_over(w.is.0, w.is.1, data);
|
||||
let (best, selection) = optimize_deflated(
|
||||
&is_family, "total_pips", DEFLATION_N_RESAMPLES, DEFLATION_BLOCK_LEN, DEFLATION_SEED,
|
||||
).expect("total_pips is a known metric");
|
||||
let (is_family, lattice) = sweep_over(w.is.0, w.is.1, data);
|
||||
let (best, selection) = match select_winner(&is_family, "total_pips", select, lattice.as_deref()) {
|
||||
Ok(v) => v,
|
||||
Err(msg) => { eprintln!("aura: {msg}"); std::process::exit(2); }
|
||||
};
|
||||
let (oos_equity, mut oos_report) = run_oos(&best.params, w.oos.0, w.oos.1, trace, data);
|
||||
oos_report.manifest.selection = Some(selection);
|
||||
WindowRun {
|
||||
@@ -1817,10 +1867,11 @@ fn walkforward_family(
|
||||
Strategy::Stage1R => {
|
||||
let space = stage1_r_space();
|
||||
walk_forward(roller, space, |w: WindowBounds| {
|
||||
let is_family = stage1_r_sweep_over(w.is.0, w.is.1, data, grid);
|
||||
let (best, selection) = optimize_deflated(
|
||||
&is_family, "sqn_normalized", DEFLATION_N_RESAMPLES, DEFLATION_BLOCK_LEN, DEFLATION_SEED,
|
||||
).expect("sqn_normalized is a known metric");
|
||||
let (is_family, lattice) = stage1_r_sweep_over(w.is.0, w.is.1, data, grid);
|
||||
let (best, selection) = match select_winner(&is_family, "sqn_normalized", select, lattice.as_deref()) {
|
||||
Ok(v) => v,
|
||||
Err(msg) => { eprintln!("aura: {msg}"); std::process::exit(2); }
|
||||
};
|
||||
let (oos_equity, mut oos_report) = run_oos_r(&best.params, w.oos.0, w.oos.1, trace, data);
|
||||
oos_report.manifest.selection = Some(selection);
|
||||
WindowRun { chosen_params: best.params, oos_equity, oos_report }
|
||||
@@ -1838,7 +1889,7 @@ fn walkforward_family(
|
||||
|
||||
/// Sweep the built-in named grid over an in-sample window, sourcing the in-memory
|
||||
/// windowed stream. Mirrors `sweep_family`, but windowed by `[from, to]`.
|
||||
fn sweep_over(from: Timestamp, to: Timestamp, data: &DataSource) -> SweepFamily {
|
||||
fn sweep_over(from: Timestamp, to: Timestamp, data: &DataSource) -> (SweepFamily, Option<Vec<usize>>) {
|
||||
let pip = data.pip_size();
|
||||
let bp = sample_blueprint_with_sinks(pip).0;
|
||||
let space = bp.param_space();
|
||||
@@ -1851,7 +1902,7 @@ fn sweep_over(from: Timestamp, to: Timestamp, data: &DataSource) -> SweepFamily
|
||||
.axis("signals.blend.weights[0]", [1.0])
|
||||
.axis("signals.blend.weights[1]", [1.0])
|
||||
.axis("bias.scale", [0.5])
|
||||
.sweep(|point| {
|
||||
.sweep_with_lattice(|point| {
|
||||
let (bp, rx_eq, rx_ex) = sample_blueprint_with_sinks(pip);
|
||||
let mut h = bp
|
||||
.bootstrap_with_cells(point)
|
||||
@@ -1866,6 +1917,7 @@ fn sweep_over(from: Timestamp, to: Timestamp, data: &DataSource) -> SweepFamily
|
||||
metrics: summarize(&equity, &exposure),
|
||||
}
|
||||
})
|
||||
.map(|(fam, lat)| (fam, Some(lat)))
|
||||
.expect("the built-in named grid matches the sample param-space")
|
||||
}
|
||||
|
||||
@@ -1960,7 +2012,7 @@ fn walkforward_window_source(from: Timestamp, to: Timestamp) -> VecSource {
|
||||
/// helper (mirrors `sweep_report`).
|
||||
#[cfg(test)]
|
||||
fn walkforward_report() -> String {
|
||||
let result = walkforward_family(Strategy::SmaCross, None, &DataSource::Synthetic, &Stage1RGrid::default());
|
||||
let result = walkforward_family(Strategy::SmaCross, None, &DataSource::Synthetic, &Stage1RGrid::default(), Selection::Argmax);
|
||||
let mut out = String::new();
|
||||
for w in &result.windows {
|
||||
out.push_str(&w.run.oos_report.to_json());
|
||||
@@ -2128,7 +2180,7 @@ fn run_mc_r_bootstrap(data: DataSource, grid: &Stage1RGrid, block_len: usize, n_
|
||||
/// over synthetic data in a `#[cfg(test)]` unit, mirroring `mc_report` /
|
||||
/// `walkforward_report` / `sweep_report`.
|
||||
fn mc_r_bootstrap_report(data: &DataSource, grid: &Stage1RGrid, block_len: usize, n_resamples: usize, seed: u64) -> String {
|
||||
let result = walkforward_family(Strategy::Stage1R, None, data, grid);
|
||||
let result = walkforward_family(Strategy::Stage1R, None, data, grid, Selection::Argmax);
|
||||
let pooled = pooled_oos_trade_rs(&result);
|
||||
let boot = r_bootstrap(&pooled, n_resamples, block_len, seed);
|
||||
mc_r_bootstrap_json(&boot)
|
||||
@@ -2215,9 +2267,24 @@ fn runs_family(id: &str, rank: Option<&str>) {
|
||||
for report in &ordered {
|
||||
println!("{}", report.to_json());
|
||||
if let Some(sel) = &report.manifest.selection {
|
||||
match sel.overfit_probability {
|
||||
Some(p) => println!(" deflated={:.4} P(overfit)={:.4}", sel.deflated_score, p),
|
||||
None => println!(" deflated={:.4}", sel.deflated_score),
|
||||
match sel.mode {
|
||||
// `deflated_score` is `None` only on an Argmax record with no
|
||||
// deflation run (report.rs); guard it, symmetric with the
|
||||
// plateau branch, so a from-disk record cannot panic here. When
|
||||
// present (the sole producer always stamps it), the bytes are
|
||||
// unchanged.
|
||||
SelectionMode::Argmax => if let Some(deflated) = sel.deflated_score {
|
||||
match sel.overfit_probability {
|
||||
Some(p) => println!(" deflated={deflated:.4} P(overfit)={p:.4}"),
|
||||
None => println!(" deflated={deflated:.4}"),
|
||||
}
|
||||
},
|
||||
SelectionMode::PlateauMean | SelectionMode::PlateauWorst => {
|
||||
let label = if matches!(sel.mode, SelectionMode::PlateauMean) { "mean" } else { "worst" };
|
||||
if let (Some(score), Some(n)) = (sel.neighbourhood_score, sel.n_neighbours) {
|
||||
println!(" plateau({label})={score:.4} over {n} cells");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2933,8 +3000,8 @@ fn main() {
|
||||
}
|
||||
},
|
||||
["walkforward", rest @ ..] => match parse_walkforward_args(rest) {
|
||||
Ok((strategy, name, persist, choice, grid)) => {
|
||||
run_walkforward(strategy, &name, persist, DataSource::from_choice(choice), &grid)
|
||||
Ok((strategy, name, persist, choice, grid, select)) => {
|
||||
run_walkforward(strategy, &name, persist, DataSource::from_choice(choice), &grid, select)
|
||||
}
|
||||
Err(msg) => {
|
||||
eprintln!("aura: {msg}");
|
||||
@@ -2965,6 +3032,29 @@ fn main() {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn select_winner_refuses_plateau_without_a_lattice() {
|
||||
// A plateau request with no lattice (a random sweep would yield None) is
|
||||
// refused, never silently argmaxed. The refuse short-circuits before the
|
||||
// family is read, so an empty family is fine here.
|
||||
let fam = SweepFamily { space: vec![], points: vec![] };
|
||||
let err = select_winner(&fam, "total_pips", Selection::Plateau(PlateauMode::Mean), None)
|
||||
.unwrap_err();
|
||||
assert!(err.contains("requires a grid sweep"), "refuse message: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_walkforward_select_flag() {
|
||||
let argmax = parse_walkforward_args(&["--strategy", "stage1-r"]).unwrap();
|
||||
assert!(matches!(argmax.5, Selection::Argmax), "default is argmax");
|
||||
let mean = parse_walkforward_args(&["--select", "plateau:mean"]).unwrap();
|
||||
assert!(matches!(mean.5, Selection::Plateau(PlateauMode::Mean)));
|
||||
let worst = parse_walkforward_args(&["--select", "plateau:worst"]).unwrap();
|
||||
assert!(matches!(worst.5, Selection::Plateau(PlateauMode::Worst)));
|
||||
assert!(parse_walkforward_args(&["--select", "bogus"]).is_err(),
|
||||
"unknown --select token is a usage error");
|
||||
}
|
||||
|
||||
fn cmp_member(key: &str, ts: &[i64], vals: &[f64]) -> FamilyMember {
|
||||
cmp_member_win(key, ts, vals, (0, 0))
|
||||
}
|
||||
@@ -3453,7 +3543,7 @@ mod tests {
|
||||
.append_family(
|
||||
"walkforward",
|
||||
FamilyKind::WalkForward,
|
||||
&walkforward_member_reports(&walkforward_family(Strategy::SmaCross, None, &DataSource::Synthetic, &Stage1RGrid::default())),
|
||||
&walkforward_member_reports(&walkforward_family(Strategy::SmaCross, None, &DataSource::Synthetic, &Stage1RGrid::default(), Selection::Argmax)),
|
||||
)
|
||||
.expect("walkforward family");
|
||||
assert_eq!((sid.as_str(), mid.as_str(), wid.as_str()), ("sweep-0", "mc-0", "walkforward-0"));
|
||||
@@ -3955,15 +4045,23 @@ mod tests {
|
||||
/// and rejects two name flags or a `--real` missing its symbol.
|
||||
#[test]
|
||||
fn parse_walkforward_args_defaults_and_accepts_real() {
|
||||
// Selection is not PartialEq/Debug (it carries the opaque PlateauMode), so
|
||||
// the comparable fields are asserted via the 5-tuple prefix and `select`
|
||||
// separately via `matches!`.
|
||||
let (strategy, name, persist, choice, grid, select) =
|
||||
parse_walkforward_args(&[]).expect("defaults parse");
|
||||
assert_eq!(
|
||||
parse_walkforward_args(&[]),
|
||||
Ok((Strategy::SmaCross, "walkforward".to_string(), false, DataChoice::Synthetic, Stage1RGrid::default()))
|
||||
(strategy, name, persist, choice, grid),
|
||||
(Strategy::SmaCross, "walkforward".to_string(), false, DataChoice::Synthetic, Stage1RGrid::default())
|
||||
);
|
||||
assert!(matches!(select, Selection::Argmax), "default selection is argmax");
|
||||
let (strategy, name, persist, choice, grid, _) =
|
||||
parse_walkforward_args(&["--real", "EURUSD", "--trace", "w"]).expect("real/trace parse");
|
||||
assert_eq!(
|
||||
parse_walkforward_args(&["--real", "EURUSD", "--trace", "w"]),
|
||||
Ok((Strategy::SmaCross, "w".to_string(), true,
|
||||
(strategy, name, persist, choice, grid),
|
||||
(Strategy::SmaCross, "w".to_string(), true,
|
||||
DataChoice::Real { symbol: "EURUSD".to_string(), from_ms: None, to_ms: None },
|
||||
Stage1RGrid::default()))
|
||||
Stage1RGrid::default())
|
||||
);
|
||||
assert!(parse_walkforward_args(&["--name", "a", "--trace", "b"]).is_err());
|
||||
assert!(parse_walkforward_args(&["--real"]).is_err());
|
||||
@@ -3972,7 +4070,7 @@ mod tests {
|
||||
#[test]
|
||||
fn parse_walkforward_args_accepts_strategy_and_grid_flags() {
|
||||
let r = parse_walkforward_args(&["--strategy", "stage1-r", "--fast", "5,10", "--stop-k", "2.0,3.0"]);
|
||||
let (strategy, _, _, _, grid) = r.expect("valid stage1-r walkforward args");
|
||||
let (strategy, _, _, _, grid, _) = r.expect("valid stage1-r walkforward args");
|
||||
assert_eq!(strategy, Strategy::Stage1R);
|
||||
assert_eq!(grid.fast, vec![5, 10]);
|
||||
assert_eq!(grid.stop_k, vec![2.0, 3.0]);
|
||||
@@ -4128,7 +4226,7 @@ mod tests {
|
||||
// closes >= 1 trade across its windows). Guards the wiring + the non-empty
|
||||
// pooling branch the parser/primitive unit tests cannot reach; mirrors
|
||||
// `mc_report` / `walkforward_report`. Deterministic (C1).
|
||||
let result = walkforward_family(Strategy::Stage1R, None, &DataSource::Synthetic, &Stage1RGrid::default());
|
||||
let result = walkforward_family(Strategy::Stage1R, None, &DataSource::Synthetic, &Stage1RGrid::default(), Selection::Argmax);
|
||||
let pooled = pooled_oos_trade_rs(&result);
|
||||
assert!(!pooled.is_empty(), "synthetic stage1-r walk-forward must pool >= 1 OOS trade R");
|
||||
|
||||
|
||||
@@ -2387,6 +2387,130 @@ fn runs_family_rank_shows_deflated_line() {
|
||||
);
|
||||
}
|
||||
|
||||
/// Property: a stage1-r walk-forward run with `--select plateau:mean` selects the
|
||||
/// neighbourhood-smoothed winner and stamps each OOS manifest with the plateau
|
||||
/// provenance (`mode = PlateauMean`, `neighbourhood_score`, `n_neighbours`); `runs
|
||||
/// family … rank` renders the `plateau(mean)=… over N cells` line, and the
|
||||
/// deflation fields are omitted (orthogonal annotation). End-to-end witness that
|
||||
/// the opt-in selection rule reaches disk (C18) and the display path renders it.
|
||||
#[test]
|
||||
fn walkforward_plateau_select_stamps_plateau_provenance() {
|
||||
let cwd = temp_cwd("runs-plateau");
|
||||
let wf = Command::new(BIN)
|
||||
.args([
|
||||
"walkforward", "--strategy", "stage1-r", "--select", "plateau:mean",
|
||||
"--fast", "50,100", "--slow", "200,400",
|
||||
"--stop-length", "14,21", "--stop-k", "2.0,3.0",
|
||||
"--name", "wf-plat",
|
||||
])
|
||||
.current_dir(&cwd)
|
||||
.output()
|
||||
.expect("spawn walkforward --select plateau:mean");
|
||||
assert!(
|
||||
wf.status.success(),
|
||||
"walkforward exit: {:?}; stderr: {}",
|
||||
wf.status,
|
||||
String::from_utf8_lossy(&wf.stderr)
|
||||
);
|
||||
|
||||
let rank = Command::new(BIN)
|
||||
.args(["runs", "family", "wf-plat-0", "rank", "sqn_normalized"])
|
||||
.current_dir(&cwd)
|
||||
.output()
|
||||
.expect("spawn runs family wf-plat-0 rank sqn_normalized");
|
||||
assert!(rank.status.success(), "rank exit: {:?}", rank.status);
|
||||
let rank_out = String::from_utf8(rank.stdout).expect("utf-8");
|
||||
let _ = std::fs::remove_dir_all(&cwd);
|
||||
|
||||
assert!(rank_out.contains("plateau(mean)="), "rank output missing plateau(mean)=: {rank_out:?}");
|
||||
assert!(rank_out.contains(" cells"), "rank output missing 'over N cells': {rank_out:?}");
|
||||
assert!(rank_out.contains("\"mode\":\"PlateauMean\""), "manifest carries PlateauMean: {rank_out:?}");
|
||||
assert!(!rank_out.contains("\"deflated_score\""), "plateau run omits deflated_score: {rank_out:?}");
|
||||
}
|
||||
|
||||
/// Property (C23, the opt-in feature's headline guarantee): `--select argmax` is a
|
||||
/// no-op against the default. A stage1-r walk-forward run with an EXPLICIT
|
||||
/// `--select argmax` produces stdout byte-for-byte identical to the same run with
|
||||
/// no `--select` flag at all, AND keeps the trials-deflation provenance
|
||||
/// (`mode == Argmax`, the deflated annotation) on each OOS manifest. Pins both
|
||||
/// halves of "plateau is strictly opt-in": the flag's default IS argmax (no
|
||||
/// divergence), and threading the new `Selection` did not silently demote argmax to
|
||||
/// a bare pick (the #144 deflation path survives). A regression where `--select`
|
||||
/// perturbed the default path, or where the default switched away from argmax,
|
||||
/// passes every other test but fails this byte-equality.
|
||||
#[test]
|
||||
fn walkforward_select_argmax_is_byte_identical_to_default() {
|
||||
let run = |args: &[&str]| {
|
||||
let cwd = temp_cwd("wf-argmax-noop");
|
||||
let out = Command::new(BIN)
|
||||
.arg("walkforward")
|
||||
.args(args)
|
||||
.current_dir(&cwd)
|
||||
.output()
|
||||
.expect("spawn aura walkforward");
|
||||
let _ = std::fs::remove_dir_all(&cwd);
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"walkforward exit: {:?}; stderr: {}",
|
||||
out.status,
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
String::from_utf8(out.stdout).expect("utf-8")
|
||||
};
|
||||
let default = run(&["--strategy", "stage1-r"]);
|
||||
let explicit = run(&["--strategy", "stage1-r", "--select", "argmax"]);
|
||||
assert_eq!(default, explicit, "explicit --select argmax must be a no-op vs the default");
|
||||
// argmax stayed the deflation path (not a bare argmax): the per-window member
|
||||
// JSON carries the Argmax mode and its deflation annotation.
|
||||
assert!(default.contains("\"mode\":\"Argmax\""), "argmax manifest mode: {default:?}");
|
||||
assert!(default.contains("\"deflated_score\""), "argmax keeps the deflation annotation: {default:?}");
|
||||
assert!(!default.contains("\"neighbourhood_score\""), "argmax omits the plateau annotation: {default:?}");
|
||||
}
|
||||
|
||||
/// Property: `--select plateau:worst` reaches disk and the display path through the
|
||||
/// DISTINCT worst-case branch — a separate `SelectionMode::PlateauWorst` enum
|
||||
/// variant on the wire and a separate `plateau(worst)=` display label. The landed
|
||||
/// happy-path E2E exercises only `plateau:mean`; the worst arm's serialization
|
||||
/// variant and its display label are otherwise untouched end-to-end, so a
|
||||
/// regression that mislabelled worst as mean (or failed to round-trip the variant)
|
||||
/// would pass the mean test. Same fixture grid as the mean E2E, only the rule
|
||||
/// differs.
|
||||
#[test]
|
||||
fn walkforward_plateau_worst_stamps_worst_variant_and_label() {
|
||||
let cwd = temp_cwd("runs-plateau-worst");
|
||||
let wf = Command::new(BIN)
|
||||
.args([
|
||||
"walkforward", "--strategy", "stage1-r", "--select", "plateau:worst",
|
||||
"--fast", "50,100", "--slow", "200,400",
|
||||
"--stop-length", "14,21", "--stop-k", "2.0,3.0",
|
||||
"--name", "wf-worst",
|
||||
])
|
||||
.current_dir(&cwd)
|
||||
.output()
|
||||
.expect("spawn walkforward --select plateau:worst");
|
||||
assert!(
|
||||
wf.status.success(),
|
||||
"walkforward exit: {:?}; stderr: {}",
|
||||
wf.status,
|
||||
String::from_utf8_lossy(&wf.stderr)
|
||||
);
|
||||
|
||||
let rank = Command::new(BIN)
|
||||
.args(["runs", "family", "wf-worst-0", "rank", "sqn_normalized"])
|
||||
.current_dir(&cwd)
|
||||
.output()
|
||||
.expect("spawn runs family wf-worst-0 rank sqn_normalized");
|
||||
assert!(rank.status.success(), "rank exit: {:?}", rank.status);
|
||||
let rank_out = String::from_utf8(rank.stdout).expect("utf-8");
|
||||
let _ = std::fs::remove_dir_all(&cwd);
|
||||
|
||||
assert!(rank_out.contains("plateau(worst)="), "rank output missing plateau(worst)=: {rank_out:?}");
|
||||
assert!(rank_out.contains("\"mode\":\"PlateauWorst\""), "manifest carries PlateauWorst: {rank_out:?}");
|
||||
// the worst label must not be rendered or stamped as mean
|
||||
assert!(!rank_out.contains("plateau(mean)="), "worst run must not render the mean label: {rank_out:?}");
|
||||
assert!(!rank_out.contains("\"mode\":\"PlateauMean\""), "worst run must not stamp PlateauMean: {rank_out:?}");
|
||||
}
|
||||
|
||||
/// Property: the bare `aura walkforward` (default SMA-cross) summary is preserved
|
||||
/// byte-for-byte by the new `--strategy`/grid plumbing — it still carries exactly
|
||||
/// `windows`, `stitched_total_pips`, `param_stability` and NOT the stage1-r-only
|
||||
|
||||
@@ -397,8 +397,20 @@ impl SweepBinder {
|
||||
}
|
||||
|
||||
/// Resolve the named axes against `param_space()` into a positional grid and
|
||||
/// run the disjoint sweep.
|
||||
/// run the disjoint sweep. `sweep` is [`SweepBinder::sweep_with_lattice`] with
|
||||
/// the lattice dropped, so every existing caller is byte-unchanged.
|
||||
pub fn sweep<F>(self, run_one: F) -> Result<SweepFamily, BindError>
|
||||
where
|
||||
F: Fn(&[Cell]) -> RunReport + Sync,
|
||||
{
|
||||
self.sweep_with_lattice(run_one).map(|(family, _lattice)| family)
|
||||
}
|
||||
|
||||
/// As [`SweepBinder::sweep`], plus the grid's per-axis radixes (`axis_lens`,
|
||||
/// in `param_space()` / odometer order). The lattice is what a plateau
|
||||
/// neighbourhood walk needs (cycle 0077); only the engine's post-`resolve_axes`
|
||||
/// grid holds it in the correct order.
|
||||
pub fn sweep_with_lattice<F>(self, run_one: F) -> Result<(SweepFamily, Vec<usize>), BindError>
|
||||
where
|
||||
F: Fn(&[Cell]) -> RunReport + Sync,
|
||||
{
|
||||
@@ -407,7 +419,8 @@ impl SweepBinder {
|
||||
let ordered = resolve_axes(&space, &self.axes)?;
|
||||
let grid = GridSpace::new(&space, ordered)
|
||||
.expect("named layer pre-validates arity/kind/non-empty");
|
||||
Ok(sweep(&grid, run_one))
|
||||
let lattice = grid.axis_lens();
|
||||
Ok((sweep(&grid, run_one), lattice))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -938,6 +951,19 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sweep_with_lattice_surfaces_grid_radixes_in_param_space_order() {
|
||||
let bp = composite_sma_cross_harness().0;
|
||||
let (fam, lattice) = bp
|
||||
.axis("sma_cross.fast.length", vec![Scalar::i64(2), Scalar::i64(3)]) // 2 values
|
||||
.axis("sma_cross.slow.length", vec![Scalar::i64(4), Scalar::i64(5)]) // 2 values
|
||||
.axis("bias.scale", vec![Scalar::f64(0.5)]) // 1 value
|
||||
.sweep_with_lattice(run_point)
|
||||
.expect("named binding resolves and runs");
|
||||
assert_eq!(lattice, vec![2, 2, 1], "radixes in param_space slot order");
|
||||
assert_eq!(lattice.iter().product::<usize>(), fam.points.len()); // 4
|
||||
}
|
||||
|
||||
/// Property (the reason `RandomBinder` exists): building a random sweep **by
|
||||
/// name** is order-independent and binds the same knob to the same range
|
||||
/// regardless of `.range(...)` call order — exactly the safety the grid's
|
||||
|
||||
@@ -39,30 +39,46 @@ pub struct RunMetrics {
|
||||
}
|
||||
|
||||
/// Which selection objective produced the record (additive provenance, C23).
|
||||
/// `Argmax` is the bare-best pick (cycle 0076). The enum is deliberately left
|
||||
/// open: cycle 0145 adds a `Plateau*` variant on this same field, so peak-vs-
|
||||
/// plateau runs stay distinguishable.
|
||||
/// `Argmax` is the bare-best pick (cycle 0076), deflated for the number of trials.
|
||||
/// `PlateauMean` / `PlateauWorst` (cycle 0077) argmax the neighbourhood-smoothed
|
||||
/// surface instead, so peak-vs-plateau runs stay distinguishable on this field.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
pub enum SelectionMode {
|
||||
Argmax,
|
||||
PlateauMean,
|
||||
PlateauWorst,
|
||||
}
|
||||
|
||||
/// Selection-provenance for a sweep winner: how its metric was deflated for the
|
||||
/// number of configurations it beat. Additive — recorded, never re-ranking (C23).
|
||||
/// `overfit_probability` is the empirical data-snooping p-value (R arm only);
|
||||
/// `None` on the `total_pips` dispersion-floor arm.
|
||||
/// Selection-provenance for a sweep winner. The selection RULE (`mode`) and its
|
||||
/// ANNOTATION are orthogonal: `Argmax` carries the trials-deflation annotation
|
||||
/// (`deflated_score` / `overfit_probability` / `n_resamples` / `block_len` /
|
||||
/// `seed`); `Plateau*` carries the smoothing annotation (`neighbourhood_score` /
|
||||
/// `n_neighbours`). Each annotation is `Option`, present iff its rule produced the
|
||||
/// record; all are additive — recorded, never re-ranking (C23). A legacy line
|
||||
/// (pre-0077) carries the deflation scalars as bare values that deserialize to
|
||||
/// `Some` (serde default), so it still loads (C14/C18).
|
||||
#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct FamilySelection {
|
||||
pub selection_metric: String,
|
||||
pub n_trials: usize,
|
||||
pub raw_winner_metric: f64,
|
||||
pub deflated_score: f64,
|
||||
pub mode: SelectionMode,
|
||||
// deflation annotation (present iff mode == Argmax with a deflation run)
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub deflated_score: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub overfit_probability: Option<f64>,
|
||||
pub mode: SelectionMode,
|
||||
pub n_resamples: usize,
|
||||
pub block_len: usize,
|
||||
pub seed: u64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub n_resamples: Option<usize>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub block_len: Option<usize>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<u64>,
|
||||
// plateau annotation (present iff mode is Plateau*)
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub neighbourhood_score: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub n_neighbours: Option<usize>,
|
||||
}
|
||||
|
||||
/// R-based signal-quality metrics (Stage-1), reduced from a `PositionManagement` dense
|
||||
@@ -763,8 +779,10 @@ mod tests {
|
||||
fn family_selection_round_trips_on_the_manifest() {
|
||||
let sel = FamilySelection {
|
||||
selection_metric: "sqn_normalized".into(), n_trials: 4, raw_winner_metric: 1.83,
|
||||
deflated_score: 0.21, overfit_probability: Some(0.06), mode: SelectionMode::Argmax,
|
||||
n_resamples: 1000, block_len: 5, seed: 42,
|
||||
mode: SelectionMode::Argmax,
|
||||
deflated_score: Some(0.21), overfit_probability: Some(0.06),
|
||||
n_resamples: Some(1000), block_len: Some(5), seed: Some(42),
|
||||
neighbourhood_score: None, n_neighbours: None,
|
||||
};
|
||||
let m = RunManifest {
|
||||
commit: "c".into(), params: vec![], window: (Timestamp(0), Timestamp(0)),
|
||||
@@ -774,6 +792,25 @@ mod tests {
|
||||
assert_eq!(back.selection, Some(sel));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn family_selection_plateau_shape_round_trips() {
|
||||
let sel = FamilySelection {
|
||||
selection_metric: "sqn_normalized".into(), n_trials: 16, raw_winner_metric: 1.42,
|
||||
mode: SelectionMode::PlateauMean,
|
||||
deflated_score: None, overfit_probability: None,
|
||||
n_resamples: None, block_len: None, seed: None,
|
||||
neighbourhood_score: Some(1.27), n_neighbours: Some(5),
|
||||
};
|
||||
let json = serde_json::to_string(&sel).unwrap();
|
||||
// plateau annotation present; deflation fields omitted (skip_serializing_if)
|
||||
assert!(json.contains("\"neighbourhood_score\":1.27"), "json: {json}");
|
||||
assert!(json.contains("\"n_neighbours\":5"), "json: {json}");
|
||||
assert!(!json.contains("deflated_score"), "deflation fields omitted under plateau: {json}");
|
||||
assert!(!json.contains("n_resamples"), "json: {json}");
|
||||
let back: FamilySelection = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(back, sel);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn position_action_round_trips_through_i64() {
|
||||
for a in [PositionAction::Buy, PositionAction::Sell, PositionAction::Close] {
|
||||
|
||||
@@ -64,6 +64,13 @@ impl GridSpace {
|
||||
false
|
||||
}
|
||||
|
||||
/// Per-axis cardinalities in `param_space()` order (the odometer radixes,
|
||||
/// last-axis-fastest). `∏ axis_lens() == len()`. The lattice shape a plateau
|
||||
/// neighbourhood walks (cycle 0077).
|
||||
pub fn axis_lens(&self) -> Vec<usize> {
|
||||
self.axes.iter().map(Vec::len).collect()
|
||||
}
|
||||
|
||||
/// The cartesian product, in odometer order: the **last** axis varies
|
||||
/// fastest. Deterministic — the same grid yields the same point sequence.
|
||||
pub fn points(&self) -> Vec<Vec<Cell>> {
|
||||
@@ -597,6 +604,20 @@ mod tests {
|
||||
assert!(!grid.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grid_axis_lens_are_the_per_axis_radixes() {
|
||||
let space = vec![
|
||||
ParamSpec { name: "a".into(), kind: ScalarKind::I64 },
|
||||
ParamSpec { name: "b".into(), kind: ScalarKind::I64 },
|
||||
];
|
||||
let grid = GridSpace::new(&space, vec![
|
||||
vec![Scalar::i64(10), Scalar::i64(20)], // axis 0: 2 values
|
||||
vec![Scalar::i64(1), Scalar::i64(2), Scalar::i64(3)], // axis 1: 3 values
|
||||
]).expect("2x3 grid");
|
||||
assert_eq!(grid.axis_lens(), vec![2, 3]);
|
||||
assert_eq!(grid.axis_lens().iter().product::<usize>(), grid.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn arity_mismatch_is_an_error() {
|
||||
let space = i64_space(2);
|
||||
|
||||
+258
-19
@@ -155,21 +155,20 @@ fn metric_value(rep: &RunReport, m: Metric) -> f64 {
|
||||
/// deterministically without panicking. An unknown metric name is a
|
||||
/// `RegistryError::UnknownMetric`.
|
||||
///
|
||||
/// The single source of truth for "best" — both [`rank_by`] (sort by it) and
|
||||
/// [`optimize`] (argmax by it) call this, so the per-metric direction lives in
|
||||
/// exactly one place. The metric name is resolved once via `resolve_metric`; the
|
||||
/// returned closure carries the resolved `Metric`, so per-comparison work is just
|
||||
/// reading each value (`metric_value`) and the key compare.
|
||||
/// The single source of best-first *ordering* — both [`rank_by`] (sort by it) and
|
||||
/// [`optimize`] (argmax by it) call this. The per-metric direction itself lives in
|
||||
/// `higher_is_better` (which this and `optimize_plateau` both read); `metric_cmp`
|
||||
/// turns that direction into a `total_cmp` over the read values. The metric name is
|
||||
/// resolved once via `resolve_metric`; the returned closure carries the resolved
|
||||
/// `Metric`, so per-comparison work is just reading each value (`metric_value`) and
|
||||
/// the key compare.
|
||||
fn metric_cmp(metric: &str) -> Result<impl Fn(&RunReport, &RunReport) -> Ordering, RegistryError> {
|
||||
let m = resolve_metric(metric)?;
|
||||
let hib = higher_is_better(m);
|
||||
Ok(move |a: &RunReport, b: &RunReport| {
|
||||
let (va, vb) = (metric_value(a, m), metric_value(b, m));
|
||||
match m {
|
||||
// lower-is-better
|
||||
Metric::MaxDrawdown | Metric::BiasSignFlips => va.total_cmp(&vb),
|
||||
// higher-is-better
|
||||
_ => vb.total_cmp(&va),
|
||||
}
|
||||
// higher-is-better ranks the larger value first; lower-is-better the smaller.
|
||||
if hib { vb.total_cmp(&va) } else { va.total_cmp(&vb) }
|
||||
})
|
||||
}
|
||||
|
||||
@@ -206,6 +205,43 @@ fn is_r_metric(m: Metric) -> bool {
|
||||
matches!(m, Metric::Sqn | Metric::SqnNormalized | Metric::ExpectancyR | Metric::NetExpectancyR)
|
||||
}
|
||||
|
||||
/// The metric's optimisation direction: `true` if a larger value is better
|
||||
/// (`total_pips`, the four R keys), `false` for the lower-is-better keys
|
||||
/// (`max_drawdown`, `bias_sign_flips`). The single direction source — `metric_cmp`
|
||||
/// (best-first ordering) and `optimize_plateau` (smoothed argmax + worst-neighbour)
|
||||
/// both read it, so the per-metric sense lives in exactly one place.
|
||||
fn higher_is_better(m: Metric) -> bool {
|
||||
!matches!(m, Metric::MaxDrawdown | Metric::BiasSignFlips)
|
||||
}
|
||||
|
||||
/// The closed grid neighbourhood of flat index `i` over a mixed-radix lattice
|
||||
/// (`axis_lens`, last axis fastest — the `GridSpace` odometer convention): `{i}`
|
||||
/// plus each in-range ±1-per-axis cell. Pure index math, deterministic. The order
|
||||
/// is `i` first then per-axis neighbours; callers use the result only as a set and
|
||||
/// for its length, so the order is not load-bearing.
|
||||
fn closed_neighbourhood(i: usize, axis_lens: &[usize]) -> Vec<usize> {
|
||||
// decompose i to per-axis coords (last axis fastest)
|
||||
let mut coords = vec![0usize; axis_lens.len()];
|
||||
let mut rem = i;
|
||||
for k in (0..axis_lens.len()).rev() {
|
||||
coords[k] = rem % axis_lens[k];
|
||||
rem /= axis_lens[k];
|
||||
}
|
||||
// recompose a coord vector to its flat index (Horner, last axis fastest)
|
||||
let recompose = |c: &[usize]| c.iter().zip(axis_lens).fold(0usize, |flat, (&ck, &len)| flat * len + ck);
|
||||
let mut out = vec![i];
|
||||
for k in 0..axis_lens.len() {
|
||||
for delta in [-1isize, 1] {
|
||||
let ck = coords[k] as isize + delta;
|
||||
if ck < 0 || ck as usize >= axis_lens[k] { continue; }
|
||||
let mut nb = coords.clone();
|
||||
nb[k] = ck as usize;
|
||||
out.push(recompose(&nb));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn member_trade_rs(rep: &RunReport) -> &[f64] {
|
||||
rep.metrics.r.as_ref().map(|r| r.trade_rs.as_slice()).unwrap_or(&[])
|
||||
}
|
||||
@@ -255,6 +291,84 @@ fn member_sd(family: &SweepFamily, m: Metric) -> f64 {
|
||||
(vals.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / (n - 1) as f64).sqrt()
|
||||
}
|
||||
|
||||
/// Plateau-selection aggregate: how a member's grid-neighbourhood metric is
|
||||
/// reduced to one smoothed score before the argmax. `Mean` averages the closed
|
||||
/// neighbourhood; `Worst` takes the most-pessimistic neighbour by the metric's
|
||||
/// direction (biasing toward interior cells, which keep more neighbours).
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub enum PlateauMode {
|
||||
Mean,
|
||||
Worst,
|
||||
}
|
||||
|
||||
/// `optimize`'s argmax over the NEIGHBOURHOOD-SMOOTHED surface, plus its plateau
|
||||
/// provenance. Each member scores as the mean (or worst-case) of its closed grid
|
||||
/// neighbourhood's `metric_value`; the winner is the best smoothed score by the
|
||||
/// metric's own direction (earliest-odometer tie, as `optimize`). `axis_lens` are
|
||||
/// the grid radixes (`param_space()` order, last axis fastest). Pure,
|
||||
/// deterministic (C1). The returned `SweepPoint` is the winning grid CELL;
|
||||
/// `raw_winner_metric` is that cell's own metric, `neighbourhood_score` the
|
||||
/// smoothed value the argmax maximised.
|
||||
pub fn optimize_plateau(
|
||||
family: &SweepFamily, axis_lens: &[usize], metric: &str, mode: PlateauMode,
|
||||
) -> Result<(SweepPoint, FamilySelection), RegistryError> {
|
||||
let m = resolve_metric(metric)?;
|
||||
let n = family.points.len();
|
||||
debug_assert_eq!(
|
||||
axis_lens.iter().product::<usize>(), n,
|
||||
"axis_lens product must equal the family size (a valid grid lattice)",
|
||||
);
|
||||
let hib = higher_is_better(m);
|
||||
// smoothed score + closed-neighbourhood size per member, in odometer order
|
||||
let scored: Vec<(usize, f64, usize)> = (0..n)
|
||||
.map(|i| {
|
||||
let nbrs = closed_neighbourhood(i, axis_lens);
|
||||
let vals: Vec<f64> = nbrs.iter().map(|&j| metric_value(&family.points[j].report, m)).collect();
|
||||
let score = match mode {
|
||||
PlateauMode::Mean => vals.iter().sum::<f64>() / vals.len() as f64,
|
||||
PlateauMode::Worst => {
|
||||
if hib {
|
||||
vals.iter().copied().fold(f64::INFINITY, f64::min)
|
||||
} else {
|
||||
vals.iter().copied().fold(f64::NEG_INFINITY, f64::max)
|
||||
}
|
||||
}
|
||||
};
|
||||
(i, score, nbrs.len())
|
||||
})
|
||||
.collect();
|
||||
// argmax the smoothed surface by direction; earliest-odometer tie (only a
|
||||
// strictly-better later cell displaces the incumbent, as `optimize`).
|
||||
let &(wi, wscore, wn) = scored
|
||||
.iter()
|
||||
.reduce(|best, cur| {
|
||||
let better = if hib { cur.1 > best.1 } else { cur.1 < best.1 };
|
||||
if better { cur } else { best }
|
||||
})
|
||||
.expect("a SweepFamily is non-empty by construction (EmptyAxis is rejected upstream)");
|
||||
let winner = family.points[wi].clone();
|
||||
let raw = metric_value(&winner.report, m);
|
||||
Ok((
|
||||
winner,
|
||||
FamilySelection {
|
||||
selection_metric: metric.to_string(),
|
||||
n_trials: n,
|
||||
raw_winner_metric: raw,
|
||||
mode: match mode {
|
||||
PlateauMode::Mean => SelectionMode::PlateauMean,
|
||||
PlateauMode::Worst => SelectionMode::PlateauWorst,
|
||||
},
|
||||
deflated_score: None,
|
||||
overfit_probability: None,
|
||||
n_resamples: None,
|
||||
block_len: None,
|
||||
seed: None,
|
||||
neighbourhood_score: Some(wscore),
|
||||
n_neighbours: Some(wn),
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
/// `optimize`'s argmax winner PLUS its trials-deflation provenance. The returned
|
||||
/// `SweepPoint` is byte-identical to `optimize(family, metric)` (additive, C23).
|
||||
/// R arm: a centred moving-block reality-check (`overfit_probability` =
|
||||
@@ -307,8 +421,10 @@ pub fn optimize_deflated(
|
||||
|
||||
Ok((winner.clone(), FamilySelection {
|
||||
selection_metric: metric.to_string(), n_trials: k, raw_winner_metric: raw,
|
||||
deflated_score, overfit_probability, mode: SelectionMode::Argmax,
|
||||
n_resamples, block_len, seed,
|
||||
mode: SelectionMode::Argmax,
|
||||
deflated_score: Some(deflated_score), overfit_probability,
|
||||
n_resamples: Some(n_resamples), block_len: Some(block_len), seed: Some(seed),
|
||||
neighbourhood_score: None, n_neighbours: None,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -746,7 +862,7 @@ mod tests {
|
||||
let sel = optimize_deflated(&fam, "total_pips", 100, 3, 1).unwrap().1;
|
||||
assert!(sel.overfit_probability.is_none());
|
||||
assert_eq!(sel.selection_metric, "total_pips");
|
||||
assert!(sel.deflated_score <= sel.raw_winner_metric); // dispersion floor ≤ raw
|
||||
assert!(sel.deflated_score.unwrap() <= sel.raw_winner_metric); // dispersion floor ≤ raw
|
||||
}
|
||||
|
||||
/// Sibling-contract parity with `r_bootstrap`, which defines `n_resamples ==
|
||||
@@ -759,8 +875,8 @@ mod tests {
|
||||
fn optimize_deflated_zero_resamples_floors_like_r_bootstrap() {
|
||||
let fam = fixture_family_with_r();
|
||||
let sel = optimize_deflated(&fam, "expectancy_r", 0, 3, 1).unwrap().1;
|
||||
assert_eq!(sel.n_resamples, 0);
|
||||
assert_eq!(sel.deflated_score, sel.raw_winner_metric, "p95 of an empty null is 0");
|
||||
assert_eq!(sel.n_resamples, Some(0));
|
||||
assert_eq!(sel.deflated_score.unwrap(), sel.raw_winner_metric, "p95 of an empty null is 0");
|
||||
assert_eq!(sel.overfit_probability, Some(1.0), "(0+1)/(0+1) Laplace floor");
|
||||
}
|
||||
|
||||
@@ -779,8 +895,8 @@ mod tests {
|
||||
}
|
||||
}
|
||||
let sel = optimize_deflated(&fam, "expectancy_r", 500, 3, 1).unwrap().1;
|
||||
assert!(sel.deflated_score.is_finite(), "deflated_score must be finite, got {}", sel.deflated_score);
|
||||
assert_eq!(sel.deflated_score, sel.raw_winner_metric);
|
||||
assert!(sel.deflated_score.unwrap().is_finite(), "deflated_score must be finite, got {:?}", sel.deflated_score);
|
||||
assert_eq!(sel.deflated_score.unwrap(), sel.raw_winner_metric);
|
||||
assert_eq!(sel.overfit_probability, Some(1.0));
|
||||
}
|
||||
|
||||
@@ -791,7 +907,7 @@ mod tests {
|
||||
let fam = fixture_family_one_strong_edge();
|
||||
let sel = optimize_deflated(&fam, "expectancy_r", 1000, 1, 9).unwrap().1;
|
||||
assert!(sel.overfit_probability.unwrap() < 0.2, "p={:?}", sel.overfit_probability);
|
||||
assert!(sel.deflated_score > 0.0);
|
||||
assert!(sel.deflated_score.unwrap() > 0.0);
|
||||
}
|
||||
|
||||
/// Every member shares the SAME strong +R edge. An *uncentred* null (each
|
||||
@@ -856,4 +972,127 @@ mod tests {
|
||||
let m = resolve_metric("sqn_normalized").unwrap();
|
||||
assert_eq!(sel.raw_winner_metric, metric_value(&winner.report, m));
|
||||
}
|
||||
|
||||
/// A 1-D grid (7 cells, odometer order) with two end spikes and a broad middle
|
||||
/// plateau in `total_pips`. Bare argmax takes an end spike; the plateau-mean
|
||||
/// argmax takes the interior plateau centre.
|
||||
fn fixture_grid_spike_vs_plateau() -> SweepFamily {
|
||||
let pips = [80.0, 5.0, 48.0, 50.0, 49.0, 5.0, 80.0];
|
||||
SweepFamily { space: vec![], points: pips.iter().map(|&p| member(p, vec![1.0])).collect() }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn closed_neighbourhood_2x3_golden() {
|
||||
// 2x3 lattice, odometer (last axis fastest):
|
||||
// (0,0)=0 (0,1)=1 (0,2)=2
|
||||
// (1,0)=3 (1,1)=4 (1,2)=5
|
||||
let sorted = |i: usize| { let mut v = closed_neighbourhood(i, &[2, 3]); v.sort_unstable(); v };
|
||||
assert_eq!(sorted(0), vec![0, 1, 3]); // corner (0,0): self + 2
|
||||
assert_eq!(sorted(4), vec![1, 3, 4, 5]); // interior (1,1): self + 3
|
||||
assert_eq!(sorted(2), vec![1, 2, 5]); // corner (0,2): self + 2
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plateau_picks_the_plateau_not_the_spike() {
|
||||
let fam = fixture_grid_spike_vs_plateau();
|
||||
let spike = optimize(&fam, "total_pips").unwrap();
|
||||
let (centre, sel) = optimize_plateau(&fam, &[7], "total_pips", PlateauMode::Mean).unwrap();
|
||||
assert_ne!(centre.report.metrics.total_pips, spike.report.metrics.total_pips);
|
||||
assert_eq!(centre.report.metrics.total_pips, 50.0, "plateau centre is the middle cell");
|
||||
assert_eq!(sel.mode, SelectionMode::PlateauMean);
|
||||
assert_eq!(sel.raw_winner_metric, 50.0);
|
||||
assert!(sel.neighbourhood_score.unwrap() < spike.report.metrics.total_pips,
|
||||
"smoothed score is below the spike's raw peak");
|
||||
assert_eq!(sel.n_neighbours, Some(3), "interior cell has self + 2 neighbours");
|
||||
assert!(sel.deflated_score.is_none() && sel.n_resamples.is_none(),
|
||||
"deflation fields omitted under plateau");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plateau_worst_is_more_conservative_than_mean() {
|
||||
let fam = fixture_grid_spike_vs_plateau();
|
||||
let (_, mean_sel) = optimize_plateau(&fam, &[7], "total_pips", PlateauMode::Mean).unwrap();
|
||||
let (_, worst_sel) = optimize_plateau(&fam, &[7], "total_pips", PlateauMode::Worst).unwrap();
|
||||
// Worst scores each cell by its weakest neighbour, never above the mean.
|
||||
assert!(worst_sel.neighbourhood_score.unwrap() <= mean_sel.neighbourhood_score.unwrap());
|
||||
assert_eq!(worst_sel.mode, SelectionMode::PlateauWorst);
|
||||
assert!(worst_sel.n_neighbours.unwrap() >= 2, "winner is an interior cell");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plateau_single_member_equals_optimize() {
|
||||
let fam = SweepFamily { space: vec![], points: vec![member(42.0, vec![1.0])] };
|
||||
let plain = optimize(&fam, "total_pips").unwrap();
|
||||
let (winner, sel) = optimize_plateau(&fam, &[1], "total_pips", PlateauMode::Mean).unwrap();
|
||||
assert_eq!(winner.report.metrics.total_pips, plain.report.metrics.total_pips);
|
||||
assert_eq!(sel.n_neighbours, Some(1), "a 1-cell grid's closed neighbourhood is itself");
|
||||
assert_eq!(sel.neighbourhood_score, Some(42.0));
|
||||
assert_eq!(sel.raw_winner_metric, 42.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optimize_plateau_is_deterministic() {
|
||||
let fam = fixture_grid_spike_vs_plateau();
|
||||
let a = optimize_plateau(&fam, &[7], "total_pips", PlateauMode::Worst).unwrap().1;
|
||||
let b = optimize_plateau(&fam, &[7], "total_pips", PlateauMode::Worst).unwrap().1;
|
||||
assert_eq!(a, b);
|
||||
}
|
||||
|
||||
/// A 1-D grid (7 cells, odometer order) in `max_drawdown` — the canonical
|
||||
/// lower-is-better key. Two end *dips* (drawdown 1.0, the lure for a bare
|
||||
/// lower-is-better argmax) flank a broad middle plateau of moderately-low
|
||||
/// drawdown (≈30), with worse shoulders. A member carries only `max_drawdown`;
|
||||
/// `total_pips`/`r` are inert here.
|
||||
fn fixture_grid_dip_vs_plateau_lower_is_better() -> SweepFamily {
|
||||
let dds = [1.0, 60.0, 30.0, 30.0, 31.0, 60.0, 1.0];
|
||||
let point = |dd: f64| SweepPoint {
|
||||
params: vec![],
|
||||
report: report_with(0.0, dd, 0),
|
||||
};
|
||||
SweepFamily { space: vec![], points: dds.iter().map(|&dd| point(dd)).collect() }
|
||||
}
|
||||
|
||||
/// Direction-flip coverage: on a lower-is-better metric (`max_drawdown`),
|
||||
/// plateau selection exercises the `else` (smaller-is-better) argmax branch AND
|
||||
/// the `Worst` arm's `NEG_INFINITY`/`max` fold. Bare argmax takes an end dip
|
||||
/// (the global minimum drawdown); the plateau-mean argmax takes the interior
|
||||
/// plateau centre, whose neighbourhood mean is the lowest. This is the
|
||||
/// lower-is-better companion the higher-is-better plateau tests do not reach.
|
||||
#[test]
|
||||
fn plateau_lower_is_better_picks_the_plateau_not_the_dip() {
|
||||
let fam = fixture_grid_dip_vs_plateau_lower_is_better();
|
||||
let dip = optimize(&fam, "max_drawdown").unwrap();
|
||||
assert_eq!(dip.report.metrics.max_drawdown, 1.0, "bare argmax takes the end dip");
|
||||
|
||||
let (centre, mean_sel) =
|
||||
optimize_plateau(&fam, &[7], "max_drawdown", PlateauMode::Mean).unwrap();
|
||||
// the plateau centre is an interior cell, not the end dip
|
||||
assert_ne!(centre.report.metrics.max_drawdown, 1.0);
|
||||
assert_eq!(mean_sel.mode, SelectionMode::PlateauMean);
|
||||
assert_eq!(mean_sel.n_neighbours, Some(3), "interior cell has self + 2 neighbours");
|
||||
// the winning cell's neighbourhood mean beats the end dip's (whose 60.0
|
||||
// shoulder drags its mean up), so the smaller-is-better argmax did not pick
|
||||
// the dip.
|
||||
assert!(
|
||||
mean_sel.neighbourhood_score.unwrap() < (1.0 + 60.0) / 2.0,
|
||||
"plateau mean is below the end dip's neighbourhood mean; score={:?}",
|
||||
mean_sel.neighbourhood_score,
|
||||
);
|
||||
|
||||
// Worst arm: each cell scores by its highest (worst) drawdown neighbour
|
||||
// (the NEG_INFINITY/max fold). It is never below the mean for a
|
||||
// lower-is-better metric, and still avoids the dip whose worst neighbour is
|
||||
// 60.0.
|
||||
let (_, worst_sel) =
|
||||
optimize_plateau(&fam, &[7], "max_drawdown", PlateauMode::Worst).unwrap();
|
||||
assert_eq!(worst_sel.mode, SelectionMode::PlateauWorst);
|
||||
assert!(
|
||||
worst_sel.neighbourhood_score.unwrap() >= mean_sel.neighbourhood_score.unwrap(),
|
||||
"worst (max drawdown) ≥ mean for a lower-is-better metric",
|
||||
);
|
||||
assert!(
|
||||
worst_sel.neighbourhood_score.unwrap() < 60.0,
|
||||
"the plateau interior's worst neighbour is below the dip's 60.0 shoulder",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user