diff --git a/crates/aura-cli/src/main.rs b/crates/aura-cli/src/main.rs index 975b4b6..ca2d6a8 100644 --- a/crates/aura-cli/src/main.rs +++ b/crates/aura-cli/src/main.rs @@ -24,7 +24,7 @@ use aura_engine::{ WalkForwardResult, WindowBounds, WindowRoller, WindowRun, }; use aura_registry::{ - group_families, mc_member_reports, optimize, rank_by, sweep_member_reports, + group_families, mc_member_reports, optimize_deflated, rank_by, sweep_member_reports, walkforward_member_reports, FamilyKind, FamilyMember, NameKind, Registry, RunTraces, TraceStore, WriteKind, }; @@ -52,6 +52,13 @@ const WF_REAL_IS_NS: i64 = 90 * WF_DAY_NS; const WF_REAL_OOS_NS: i64 = 30 * WF_DAY_NS; const WF_REAL_STEP_NS: i64 = 30 * WF_DAY_NS; +/// Trials-deflation resampling budget for walk-forward winner selection. Recorded +/// on each winner's manifest (so `overfit_probability` is reproducible by re-run); +/// CLI flags for these are a deferred refinement. +const DEFLATION_N_RESAMPLES: usize = 1000; +const DEFLATION_BLOCK_LEN: usize = 5; +const DEFLATION_SEED: u64 = 0xDEF1_A7ED; + /// The built-in synthetic price stream: rises through t=4 then reverses, so the /// demo trace carries one exposure sign flip and a real drawdown (C22 populated /// trace). Deterministic and fixed (C1). @@ -166,6 +173,7 @@ fn sim_optimal_manifest( window, seed, broker: format!("sim-optimal(pip_size={pip_size})"), + selection: None, } } @@ -1792,8 +1800,11 @@ fn walkforward_family( let space = sample_blueprint_with_sinks(data.pip_size()).0.param_space(); walk_forward(roller, space, |w: WindowBounds| { let is_family = sweep_over(w.is.0, w.is.1, data); - let best = optimize(&is_family, "total_pips").expect("total_pips is a known metric"); - let (oos_equity, oos_report) = run_oos(&best.params, w.oos.0, w.oos.1, trace, data); + let (best, selection) = optimize_deflated( + &is_family, "total_pips", DEFLATION_N_RESAMPLES, DEFLATION_BLOCK_LEN, DEFLATION_SEED, + ).expect("total_pips is a known metric"); + let (oos_equity, mut oos_report) = run_oos(&best.params, w.oos.0, w.oos.1, trace, data); + oos_report.manifest.selection = Some(selection); WindowRun { // The tag-free sweep winner is the chosen point; its kinds live on // WalkForwardResult.space (computed once above from the same blueprint). @@ -1807,8 +1818,11 @@ fn walkforward_family( let space = stage1_r_space(); walk_forward(roller, space, |w: WindowBounds| { let is_family = stage1_r_sweep_over(w.is.0, w.is.1, data, grid); - let best = optimize(&is_family, "sqn_normalized").expect("sqn_normalized is a known metric"); - let (oos_equity, oos_report) = run_oos_r(&best.params, w.oos.0, w.oos.1, trace, data); + let (best, selection) = optimize_deflated( + &is_family, "sqn_normalized", DEFLATION_N_RESAMPLES, DEFLATION_BLOCK_LEN, DEFLATION_SEED, + ).expect("sqn_normalized is a known metric"); + let (oos_equity, mut oos_report) = run_oos_r(&best.params, w.oos.0, w.oos.1, trace, data); + oos_report.manifest.selection = Some(selection); WindowRun { chosen_params: best.params, oos_equity, oos_report } }) } @@ -2200,6 +2214,12 @@ fn runs_family(id: &str, rank: Option<&str>) { }; for report in &ordered { println!("{}", report.to_json()); + if let Some(sel) = &report.manifest.selection { + match sel.overfit_probability { + Some(p) => println!(" deflated={:.4} P(overfit)={:.4}", sel.deflated_score, p), + None => println!(" deflated={:.4}", sel.deflated_score), + } + } } } diff --git a/crates/aura-cli/tests/cli_run.rs b/crates/aura-cli/tests/cli_run.rs index d958a35..bef80f2 100644 --- a/crates/aura-cli/tests/cli_run.rs +++ b/crates/aura-cli/tests/cli_run.rs @@ -2349,6 +2349,44 @@ fn walkforward_strategy_stage1_r_reports_oos_r() { assert_eq!(out, out2, "stage1-r walkforward is deterministic"); } +/// Property: a stage1-r walk-forward stamps each OOS winner's manifest with the +/// trials-deflation provenance (#144), and `runs family rank` surfaces it as a +/// human-readable `deflated=… P(overfit)=…` line beside each member's JSON. This is +/// the end-to-end witness that the selection record reaches disk (C18) and the +/// display path renders it; sweep/mc families stay unstamped (verified elsewhere). +#[test] +fn runs_family_rank_shows_deflated_line() { + let cwd = temp_cwd("runs-deflated"); + let wf = Command::new(BIN) + .args(["walkforward", "--strategy", "stage1-r", "--name", "wf-defl"]) + .current_dir(&cwd) + .output() + .expect("spawn walkforward --strategy stage1-r --name wf-defl"); + assert!( + wf.status.success(), + "walkforward exit: {:?}; stderr: {}", + wf.status, + String::from_utf8_lossy(&wf.stderr) + ); + + let rank = Command::new(BIN) + .args(["runs", "family", "wf-defl-0", "rank", "sqn_normalized"]) + .current_dir(&cwd) + .output() + .expect("spawn runs family wf-defl-0 rank sqn_normalized"); + assert!(rank.status.success(), "rank exit: {:?}", rank.status); + let rank_out = String::from_utf8(rank.stdout).expect("utf-8"); + let _ = std::fs::remove_dir_all(&cwd); + + assert!(rank_out.contains("deflated="), "rank output missing deflated=: {rank_out:?}"); + assert!(rank_out.contains("P(overfit)="), "rank output missing P(overfit)=: {rank_out:?}"); + // each stamped member's JSON also carries the selection block (selection_metric). + assert!( + rank_out.contains("\"selection_metric\":\"sqn_normalized\""), + "selection block present: {rank_out:?}" + ); +} + /// Property: the bare `aura walkforward` (default SMA-cross) summary is preserved /// byte-for-byte by the new `--strategy`/grid plumbing — it still carries exactly /// `windows`, `stitched_total_pips`, `param_stability` and NOT the stage1-r-only diff --git a/crates/aura-engine/src/blueprint.rs b/crates/aura-engine/src/blueprint.rs index c9eadcb..91b7c2b 100644 --- a/crates/aura-engine/src/blueprint.rs +++ b/crates/aura-engine/src/blueprint.rs @@ -932,6 +932,7 @@ mod tests { window: (Timestamp(0), Timestamp(0)), seed: 0, broker: "test".to_string(), + selection: None, }, metrics: summarize(&equity, &exposure), } diff --git a/crates/aura-engine/src/harness.rs b/crates/aura-engine/src/harness.rs index c26f8ec..8af001b 100644 --- a/crates/aura-engine/src/harness.rs +++ b/crates/aura-engine/src/harness.rs @@ -140,15 +140,15 @@ pub fn window_of(sources: &[Box]) -> Option<(Timestamp, Timestamp)> /// completely determines the sequence; no external entropy, no global state. /// Bit-stable across toolchains and crate versions — the property C1 needs for /// seed-as-input reproducibility (C12). -pub(crate) struct SplitMix64 { +pub struct SplitMix64 { state: u64, } impl SplitMix64 { - pub(crate) fn new(seed: u64) -> Self { + pub fn new(seed: u64) -> Self { Self { state: seed } } - pub(crate) fn next_u64(&mut self) -> u64 { + pub fn next_u64(&mut self) -> u64 { self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15); let mut z = self.state; z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); diff --git a/crates/aura-engine/src/lib.rs b/crates/aura-engine/src/lib.rs index eb1fc8b..5c74a07 100644 --- a/crates/aura-engine/src/lib.rs +++ b/crates/aura-engine/src/lib.rs @@ -58,18 +58,21 @@ pub use blueprint::{ pub use builder::{BuildError, GraphBuilder, InPort, NodeHandle, OutPort, RoleHandle}; pub use graph_model::model_to_json; pub use harness::{ - window_of, BootstrapError, Edge, FlatGraph, Harness, Source, SourceSpec, SyntheticSpec, Target, - VecSource, + window_of, BootstrapError, Edge, FlatGraph, Harness, Source, SourceSpec, SplitMix64, + SyntheticSpec, Target, VecSource, }; pub use report::{ - derive_position_events, f64_field, join_on_ts, r_metrics_from_rs, summarize, summarize_r, - ColumnarTrace, JoinedRow, PositionAction, PositionEvent, RMetrics, RunManifest, RunMetrics, - RunReport, + derive_position_events, expected_max_of_normals, f64_field, inv_norm_cdf, join_on_ts, + r_metrics_from_rs, summarize, summarize_r, ColumnarTrace, FamilySelection, JoinedRow, + PositionAction, PositionEvent, RMetrics, RunManifest, RunMetrics, RunReport, SelectionMode, }; pub use sweep::{ sweep, GridSpace, ParamRange, RandomSpace, Space, SweepError, SweepFamily, SweepPoint, }; -pub use mc::{monte_carlo, r_bootstrap, McAggregate, McDraw, McFamily, MetricStats, RBootstrap}; +pub use mc::{ + monte_carlo, r_bootstrap, resample_block, McAggregate, McDraw, McFamily, MetricStats, + RBootstrap, +}; pub use walkforward::{ param_stability, walk_forward, RollMode, WalkForwardError, WalkForwardResult, WindowBounds, WindowOutcome, WindowRoller, WindowRun, diff --git a/crates/aura-engine/src/mc.rs b/crates/aura-engine/src/mc.rs index a6edaf5..3fc5a81 100644 --- a/crates/aura-engine/src/mc.rs +++ b/crates/aura-engine/src/mc.rs @@ -166,6 +166,26 @@ pub struct RBootstrap { pub n_resamples: usize, } +/// One moving-block resample of `rs` to length `rs.len()` (non-circular; the final +/// block is truncated so the resample has exactly `n` values). `block_len` must be +/// pre-clamped to `[1, rs.len()]` by the caller. Pure given the `rng` state — the +/// shared kernel behind `r_bootstrap` and the trials-deflation reality-check (C1). +pub fn resample_block(rs: &[f64], block_len: usize, rng: &mut SplitMix64) -> Vec { + let n = rs.len(); + debug_assert!( + (1..=n).contains(&block_len), + "resample_block requires block_len in [1, rs.len()] (caller pre-clamps); \ + got block_len={block_len}, rs.len()={n}", + ); + let mut sample: Vec = Vec::with_capacity(n); + while sample.len() < n { + let start = (rng.next_u64() % (n - block_len + 1) as u64) as usize; + let take = block_len.min(n - sample.len()); + sample.extend_from_slice(&rs[start..start + take]); + } + sample +} + /// Moving-block bootstrap of `rs` (non-circular; the final block of each resample is /// truncated so the resample has exactly `n` values). `block_len` is clamped to /// `[1, n]`. Empty `rs` or zero resamples -> an all-zero `RBootstrap`. Pure given @@ -185,12 +205,7 @@ pub fn r_bootstrap(rs: &[f64], n_resamples: usize, block_len: usize, seed: u64) let mut rng = SplitMix64::new(seed); let mut means: Vec = Vec::with_capacity(n_resamples); for _ in 0..n_resamples { - let mut sample: Vec = Vec::with_capacity(n); - while sample.len() < n { - let start = (rng.next_u64() % (n - block_len + 1) as u64) as usize; - let take = block_len.min(n - sample.len()); - sample.extend_from_slice(&rs[start..start + take]); - } + let sample = resample_block(rs, block_len, &mut rng); means.push(sample.iter().sum::() / n as f64); } let e_r = MetricStats::from_values(&means); @@ -227,6 +242,7 @@ mod tests { window, seed, broker: "test".to_string(), + selection: None, }, metrics: summarize(&equity, &exposure), } @@ -250,6 +266,7 @@ mod tests { window: (Timestamp(0), Timestamp(0)), seed: i as u64, broker: "t".to_string(), + selection: None, }, metrics: RunMetrics { total_pips: v, @@ -359,6 +376,42 @@ mod tests { assert_eq!(back, s); } + #[test] + fn resample_block_returns_n_values_from_contiguous_runs() { + // The extracted kernel's standalone contract (now a public entry point): + // given a pre-clamped block_len in [1, n], it yields exactly `n` values, each + // appearing in `rs`, arranged as contiguous runs of `rs` (the final run + // truncated to reach exactly n). Distinct powers of ten make membership a sum + // check. n=5, block_len=2 -> runs of length 2, 2, 1. + let rs = [1.0, 10.0, 100.0, 1000.0, 10000.0]; + let n = rs.len(); + let block_len = 2; + let mut rng = SplitMix64::new(7); + let sample = resample_block(&rs, block_len, &mut rng); + assert_eq!(sample.len(), n, "resample has exactly n values"); + assert!( + sample.iter().all(|v| rs.contains(v)), + "every resampled value is drawn from rs", + ); + // Reconstruct the contiguous-run structure: walk the sample in blocks of + // `block_len` (last truncated) and assert each block is a contiguous slice of rs. + let mut filled = 0usize; + while filled < n { + let take = block_len.min(n - filled); + let run = &sample[filled..filled + take]; + let start = rs + .iter() + .position(|v| v == &run[0]) + .expect("run head is in rs"); + assert_eq!( + run, + &rs[start..start + take], + "each run is a contiguous slice of rs", + ); + filled += take; + } + } + #[test] fn r_bootstrap_empty_series_is_all_zero() { let b = r_bootstrap(&[], 100, 1, 7); diff --git a/crates/aura-engine/src/report.rs b/crates/aura-engine/src/report.rs index 81129a9..b0bd5e8 100644 --- a/crates/aura-engine/src/report.rs +++ b/crates/aura-engine/src/report.rs @@ -38,6 +38,33 @@ pub struct RunMetrics { pub r: Option, } +/// Which selection objective produced the record (additive provenance, C23). +/// `Argmax` is the bare-best pick (cycle 0076). The enum is deliberately left +/// open: cycle 0145 adds a `Plateau*` variant on this same field, so peak-vs- +/// plateau runs stay distinguishable. +#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub enum SelectionMode { + Argmax, +} + +/// Selection-provenance for a sweep winner: how its metric was deflated for the +/// number of configurations it beat. Additive — recorded, never re-ranking (C23). +/// `overfit_probability` is the empirical data-snooping p-value (R arm only); +/// `None` on the `total_pips` dispersion-floor arm. +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct FamilySelection { + pub selection_metric: String, + pub n_trials: usize, + pub raw_winner_metric: f64, + pub deflated_score: f64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub overfit_probability: Option, + pub mode: SelectionMode, + pub n_resamples: usize, + pub block_len: usize, + pub seed: u64, +} + /// R-based signal-quality metrics (Stage-1), reduced from a `PositionManagement` dense /// record stream by [`summarize_r`]. Account- and instrument-agnostic (pure R). Carries /// the enriched dispersion/churn fields (SQN, conviction terciles, net-of-cost). @@ -431,6 +458,11 @@ pub struct RunManifest { pub seed: u64, /// The broker profile label, e.g. `"sim-optimal(pip_size=0.0001)"`. pub broker: String, + /// Selection provenance, present only on a sweep/walk-forward winner; a + /// standalone run and every pre-0076 line read back as `None`. One-directional + /// serde widening (C14/C18), identical idiom to `RunMetrics.r`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selection: Option, } /// A run's full structured result: the descriptor plus the metrics it @@ -633,6 +665,56 @@ fn scalar_to_f64(s: Scalar) -> f64 { } } +/// Inverse standard-normal CDF (quantile function), Acklam's rational +/// approximation — absolute error < ~1.2e-9 over `p ∈ (0,1)`. Pure (C1). +/// Callers pass strictly-interior `p` (`k >= 2` keeps the argument off the +/// boundaries); the boundary branches return ±inf as a defined limit. +// Acklam's published coefficients verbatim; some carry more decimals than an +// f64 holds (clippy::excessive_precision) — kept literal as reference constants. +#[allow(clippy::excessive_precision)] +pub fn inv_norm_cdf(p: f64) -> f64 { + const A: [f64; 6] = [-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, + 1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00]; + const B: [f64; 5] = [-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, + 6.680131188771972e+01, -1.328068155288572e+01]; + const C: [f64; 6] = [-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, + -2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00]; + const D: [f64; 4] = [7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, + 3.754408661907416e+00]; + const P_LOW: f64 = 0.02425; + if p <= 0.0 { return f64::NEG_INFINITY; } + if p >= 1.0 { return f64::INFINITY; } + if p < P_LOW { + let q = (-2.0 * p.ln()).sqrt(); + (((((C[0]*q + C[1])*q + C[2])*q + C[3])*q + C[4])*q + C[5]) + / ((((D[0]*q + D[1])*q + D[2])*q + D[3])*q + 1.0) + } else if p <= 1.0 - P_LOW { + let q = p - 0.5; + let r = q * q; + (((((A[0]*r + A[1])*r + A[2])*r + A[3])*r + A[4])*r + A[5]) * q + / (((((B[0]*r + B[1])*r + B[2])*r + B[3])*r + B[4])*r + 1.0) + } else { + let q = (-2.0 * (1.0 - p).ln()).sqrt(); + -(((((C[0]*q + C[1])*q + C[2])*q + C[3])*q + C[4])*q + C[5]) + / ((((D[0]*q + D[1])*q + D[2])*q + D[3])*q + 1.0) + } +} + +/// Expected maximum of `k` i.i.d. standard normals — the hurdle a search of `k` +/// configurations clears by chance alone. `k <= 1` -> `0.0` (a single trial has no +/// multiple-comparison inflation, and guards the `Φ⁻¹(1 - 1/k) -> Φ⁻¹(0)` +/// divergence). For `k >= 2`: +/// `(1 - γ)·Φ⁻¹(1 - 1/k) + γ·Φ⁻¹(1 - 1/(k·e))`, `γ = 0.5772156649…`. Pure (C1). +pub fn expected_max_of_normals(k: usize) -> f64 { + if k <= 1 { + return 0.0; + } + const GAMMA: f64 = 0.577_215_664_901_532_9; + let kf = k as f64; + (1.0 - GAMMA) * inv_norm_cdf(1.0 - 1.0 / kf) + + GAMMA * inv_norm_cdf(1.0 - 1.0 / (kf * std::f64::consts::E)) +} + #[cfg(test)] mod tests { use super::*; @@ -641,6 +723,57 @@ mod tests { use aura_std::{Bias, Recorder, SimBroker, Sma, Sub}; use std::sync::mpsc; + #[test] + fn inv_norm_cdf_matches_known_quantiles() { + assert!((inv_norm_cdf(0.975) - 1.959964).abs() < 1e-3); + assert!(inv_norm_cdf(0.5).abs() < 1e-9); + assert!((inv_norm_cdf(0.1) + inv_norm_cdf(0.9)).abs() < 1e-9); // symmetry + } + + #[test] + fn expected_max_of_normals_is_zero_at_one_and_monotone() { + assert_eq!(expected_max_of_normals(1), 0.0); + assert_eq!(expected_max_of_normals(0), 0.0); + let (e2, e4, e10, e100) = ( + expected_max_of_normals(2), expected_max_of_normals(4), + expected_max_of_normals(10), expected_max_of_normals(100), + ); + assert!(e2 < e4 && e4 < e10 && e10 < e100); // strictly increasing + assert!((1.0..1.1).contains(&e4)); // ~1.05 (not the √(2 ln 4)=1.66 asymptote) + assert!((2.4..2.7).contains(&e100)); // ~2.53 + } + + #[test] + fn runmanifest_without_selection_field_deserialises_to_none() { + let json = r#"{"commit":"c","params":[],"window":[0,0],"seed":0,"broker":"b"}"#; + let m: RunManifest = serde_json::from_str(json).expect("legacy manifest loads"); + assert!(m.selection.is_none()); + } + + #[test] + fn runmanifest_without_selection_serialises_without_the_key() { + let m = RunManifest { + commit: "c".into(), params: vec![], window: (Timestamp(0), Timestamp(0)), + seed: 0, broker: "b".into(), selection: None, + }; + assert!(!serde_json::to_string(&m).unwrap().contains("selection")); + } + + #[test] + fn family_selection_round_trips_on_the_manifest() { + let sel = FamilySelection { + selection_metric: "sqn_normalized".into(), n_trials: 4, raw_winner_metric: 1.83, + deflated_score: 0.21, overfit_probability: Some(0.06), mode: SelectionMode::Argmax, + n_resamples: 1000, block_len: 5, seed: 42, + }; + let m = RunManifest { + commit: "c".into(), params: vec![], window: (Timestamp(0), Timestamp(0)), + seed: 0, broker: "b".into(), selection: Some(sel.clone()), + }; + let back: RunManifest = serde_json::from_str(&serde_json::to_string(&m).unwrap()).unwrap(); + assert_eq!(back.selection, Some(sel)); + } + #[test] fn position_action_round_trips_through_i64() { for a in [PositionAction::Buy, PositionAction::Sell, PositionAction::Close] { @@ -785,6 +918,7 @@ mod tests { window: (Timestamp(1), Timestamp(5)), seed: 0, broker: "sim-optimal(pip_size=0.0001)".to_string(), + selection: None, }, metrics, } @@ -1160,6 +1294,7 @@ mod tests { window: (Timestamp(1), Timestamp(6)), seed: 0, broker: "sim-optimal(pip_size=1.0)".to_string(), + selection: None, }, metrics: RunMetrics { total_pips: 12.0, @@ -1188,6 +1323,7 @@ mod tests { window: (Timestamp(1), Timestamp(6)), seed: 0, broker: "sim-optimal(pip_size=1.0)".to_string(), + selection: None, }, metrics: RunMetrics { total_pips: 12.0, max_drawdown: 1.0, bias_sign_flips: 1, r: None }, }; @@ -1208,6 +1344,7 @@ mod tests { window: (Timestamp(1), Timestamp(6)), seed: 0, broker: "sim-optimal(pip_size=1.0)".to_string(), + selection: None, }, metrics: RunMetrics { total_pips: 12.0, max_drawdown: 1.0, bias_sign_flips: 1, r: None }, }; diff --git a/crates/aura-engine/src/sweep.rs b/crates/aura-engine/src/sweep.rs index 93f3be9..aa34f7f 100644 --- a/crates/aura-engine/src/sweep.rs +++ b/crates/aura-engine/src/sweep.rs @@ -646,6 +646,7 @@ mod tests { window: (Timestamp(0), Timestamp(0)), seed: 0, broker: "test".to_string(), + selection: None, }, metrics: summarize(&equity, &exposure), } diff --git a/crates/aura-engine/src/walkforward.rs b/crates/aura-engine/src/walkforward.rs index 3628dc4..52cf559 100644 --- a/crates/aura-engine/src/walkforward.rs +++ b/crates/aura-engine/src/walkforward.rs @@ -291,6 +291,7 @@ mod tests { window: (Timestamp(0), Timestamp(0)), seed: 0, broker: "t".to_string(), + selection: None, }, metrics: summarize(&[], &[]), } diff --git a/crates/aura-engine/tests/random_sweep_e2e.rs b/crates/aura-engine/tests/random_sweep_e2e.rs index 6799520..1f596b3 100644 --- a/crates/aura-engine/tests/random_sweep_e2e.rs +++ b/crates/aura-engine/tests/random_sweep_e2e.rs @@ -115,6 +115,7 @@ fn run_point(point: &[Cell]) -> RunReport { window: (Timestamp(0), Timestamp(0)), seed: 0, broker: "test".to_string(), + selection: None, }, metrics: summarize(&equity, &exposure), } diff --git a/crates/aura-ingest/examples/ger40_breakout_sweep.rs b/crates/aura-ingest/examples/ger40_breakout_sweep.rs index 7c75001..d70bf60 100644 --- a/crates/aura-ingest/examples/ger40_breakout_sweep.rs +++ b/crates/aura-ingest/examples/ger40_breakout_sweep.rs @@ -66,6 +66,7 @@ fn run_point(server: &Arc, point: &[Cell], from: Timestamp, to: Time window: (from, to), seed: 0, broker: "sim-optimal(pip_size=1)".to_string(), + selection: None, }, metrics: summarize(&equity, &exposure), } diff --git a/crates/aura-ingest/examples/ger40_breakout_walkforward.rs b/crates/aura-ingest/examples/ger40_breakout_walkforward.rs index 2c266f5..925dfc5 100644 --- a/crates/aura-ingest/examples/ger40_breakout_walkforward.rs +++ b/crates/aura-ingest/examples/ger40_breakout_walkforward.rs @@ -76,6 +76,7 @@ fn run_point( window: (from, to), seed: 0, broker: "sim-optimal(pip_size=1)".to_string(), + selection: None, }, metrics: summarize(&equity, &exposure), }; diff --git a/crates/aura-ingest/examples/shared/breakout_real.rs b/crates/aura-ingest/examples/shared/breakout_real.rs index 5cb9862..94c7b44 100644 --- a/crates/aura-ingest/examples/shared/breakout_real.rs +++ b/crates/aura-ingest/examples/shared/breakout_real.rs @@ -399,6 +399,7 @@ pub fn report_from_trace(trace: &[BarTrace], from: Timestamp, to: Timestamp) -> window: (from, to), seed: 0, broker: "sim-optimal(pip_size=1)".to_string(), + selection: None, }, metrics, } diff --git a/crates/aura-ingest/tests/ger40_breakout_world.rs b/crates/aura-ingest/tests/ger40_breakout_world.rs index 30b29e9..f5c6a15 100644 --- a/crates/aura-ingest/tests/ger40_breakout_world.rs +++ b/crates/aura-ingest/tests/ger40_breakout_world.rs @@ -68,6 +68,7 @@ fn run_point(server: &Arc, point: &[Cell], from: Timestamp, to: Time window: (from, to), seed: 0, broker: "sim-optimal(pip_size=1)".to_string(), + selection: None, }, metrics: summarize(&equity, &exposure), } diff --git a/crates/aura-ingest/tests/real_bars.rs b/crates/aura-ingest/tests/real_bars.rs index af651e0..d58d4f5 100644 --- a/crates/aura-ingest/tests/real_bars.rs +++ b/crates/aura-ingest/tests/real_bars.rs @@ -89,6 +89,7 @@ fn run_sample_over(prices: Vec<(Timestamp, Scalar)>) -> RunReport { window, seed: 0, broker: "sim-optimal(pip_size=0.0001)".to_string(), + selection: None, }, metrics, } diff --git a/crates/aura-ingest/tests/streaming_seam.rs b/crates/aura-ingest/tests/streaming_seam.rs index f81cc38..2977b9e 100644 --- a/crates/aura-ingest/tests/streaming_seam.rs +++ b/crates/aura-ingest/tests/streaming_seam.rs @@ -89,6 +89,7 @@ fn run_sample(window: (Timestamp, Timestamp), source: Box) -> RunRep window, seed: 0, broker: "sim-optimal(pip_size=0.0001)".to_string(), + selection: None, }, metrics: summarize(&equity, &exposure), } diff --git a/crates/aura-registry/src/compat.rs b/crates/aura-registry/src/compat.rs index 374991c..4926172 100644 --- a/crates/aura-registry/src/compat.rs +++ b/crates/aura-registry/src/compat.rs @@ -10,7 +10,7 @@ //! (`Registry::append`, `RunReport::to_json`) is untouched and still emits the //! tagged form, so this is a one-directional widening, not a format change. -use aura_engine::{RunManifest, RunMetrics, RunReport, Scalar, Timestamp}; +use aura_engine::{FamilySelection, RunManifest, RunMetrics, RunReport, Scalar, Timestamp}; use serde::de::{self, Deserializer}; use serde::Deserialize; @@ -30,6 +30,8 @@ struct RunManifestRead { window: (Timestamp, Timestamp), seed: u64, broker: String, + #[serde(default)] + selection: Option, } /// A param value that accepts BOTH the current tagged [`Scalar`] form @@ -62,7 +64,7 @@ impl<'de> Deserialize<'de> for ScalarRead { impl From for RunReport { fn from(r: RunReportRead) -> Self { - let RunManifestRead { commit, params, window, seed, broker } = r.manifest; + let RunManifestRead { commit, params, window, seed, broker, selection } = r.manifest; RunReport { manifest: RunManifest { commit, @@ -70,6 +72,7 @@ impl From for RunReport { window, seed, broker, + selection, }, metrics: r.metrics, } diff --git a/crates/aura-registry/src/lib.rs b/crates/aura-registry/src/lib.rs index dd367a8..8a2e595 100644 --- a/crates/aura-registry/src/lib.rs +++ b/crates/aura-registry/src/lib.rs @@ -17,7 +17,10 @@ use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; -use aura_engine::{RunReport, SweepFamily, SweepPoint}; +use aura_engine::{ + expected_max_of_normals, r_metrics_from_rs, resample_block, FamilySelection, MetricStats, + RunReport, SelectionMode, SplitMix64, SweepFamily, SweepPoint, +}; mod compat; @@ -107,25 +110,10 @@ enum Metric { NetExpectancyR, } -/// The per-metric **best-first ordering** of two reports: `Less` means `a` is -/// better than `b`. "Best" is fixed by each metric's meaning: `total_pips` and -/// the four R keys (`sqn`, `sqn_normalized`, `expectancy_r`, `net_expectancy_r`) higher-is-better; -/// `max_drawdown` and `bias_sign_flips` lower-is-better. The top-level `f64` keys -/// (`total_pips`, `max_drawdown`) use `partial_cmp` — finite by construction, so -/// the `Ordering` always exists. The R keys live in an optional `metrics.r` block -/// and use `total_cmp`: a member with `r: None` injects `f64::NEG_INFINITY` (the -/// worst rank for higher-is-better), and `total_cmp` orders that — and any stray -/// `NaN` — deterministically without panicking. An unknown metric name is a -/// `RegistryError::UnknownMetric`. -/// -/// The single source of truth for "best" — both [`rank_by`] (sort by it) and -/// [`optimize`] (argmax by it) call this, so the per-metric direction lives in -/// exactly one place. The metric name is matched once here; the returned closure -/// carries the resolved `Metric`, so per-comparison work is just the key compare. -fn metric_cmp( - metric: &str, -) -> Result Ordering, RegistryError> { - let metric = match metric { +/// Resolve a metric NAME to the closed `Metric` kind (the single name→kind map). +/// An unknown name is a `RegistryError::UnknownMetric`. +fn resolve_metric(name: &str) -> Result { + Ok(match name { "total_pips" => Metric::TotalPips, "max_drawdown" => Metric::MaxDrawdown, // accept the new name AND the pre-rename one (CLI back-compat). @@ -135,31 +123,52 @@ fn metric_cmp( "expectancy_r" => Metric::ExpectancyR, "net_expectancy_r" => Metric::NetExpectancyR, other => return Err(RegistryError::UnknownMetric(other.to_string())), - }; - // The R metrics live in an optional block; a missing `r` sorts to the bottom - // for these higher-is-better keys (NEG_INFINITY < any finite R). + }) +} + +/// The metric's scalar value for one report. R metrics live in the optional `r` +/// block; a missing block reads `NEG_INFINITY` (the worst rank for the +/// higher-is-better R keys), preserving `metric_cmp`'s documented contract. +fn metric_value(rep: &RunReport, m: Metric) -> f64 { fn r_get(rep: &RunReport, f: impl Fn(&aura_engine::RMetrics) -> f64) -> f64 { rep.metrics.r.as_ref().map(f).unwrap_or(f64::NEG_INFINITY) } - Ok(move |a: &RunReport, b: &RunReport| match metric { - // higher-is-better -> better when greater - Metric::TotalPips => b.metrics.total_pips.partial_cmp(&a.metrics.total_pips).unwrap(), - // lower-is-better -> better when smaller - Metric::MaxDrawdown => a.metrics.max_drawdown.partial_cmp(&b.metrics.max_drawdown).unwrap(), - Metric::BiasSignFlips => { - a.metrics.bias_sign_flips.cmp(&b.metrics.bias_sign_flips) - } - // higher-is-better; total_cmp so NEG_INFINITY orders deterministically and - // no NaN can panic. - Metric::Sqn => r_get(b, |r| r.sqn).total_cmp(&r_get(a, |r| r.sqn)), - Metric::SqnNormalized => { - r_get(b, |r| r.sqn_normalized).total_cmp(&r_get(a, |r| r.sqn_normalized)) - } - Metric::ExpectancyR => { - r_get(b, |r| r.expectancy_r).total_cmp(&r_get(a, |r| r.expectancy_r)) - } - Metric::NetExpectancyR => { - r_get(b, |r| r.net_expectancy_r).total_cmp(&r_get(a, |r| r.net_expectancy_r)) + match m { + Metric::TotalPips => rep.metrics.total_pips, + Metric::MaxDrawdown => rep.metrics.max_drawdown, + Metric::BiasSignFlips => rep.metrics.bias_sign_flips as f64, + Metric::Sqn => r_get(rep, |r| r.sqn), + Metric::SqnNormalized => r_get(rep, |r| r.sqn_normalized), + Metric::ExpectancyR => r_get(rep, |r| r.expectancy_r), + Metric::NetExpectancyR => r_get(rep, |r| r.net_expectancy_r), + } +} + +/// The per-metric **best-first ordering** of two reports: `Less` means `a` is +/// better than `b`. "Best" is fixed by each metric's meaning: `total_pips` and +/// the four R keys (`sqn`, `sqn_normalized`, `expectancy_r`, `net_expectancy_r`) +/// higher-is-better; `max_drawdown` and `bias_sign_flips` lower-is-better. Every +/// metric compares via `total_cmp` (a total order over `f64`): top-level keys are +/// finite by construction, while the R keys live in an optional `metrics.r` block +/// where a member with `r: None` reads `f64::NEG_INFINITY` (the worst rank for +/// higher-is-better) — `total_cmp` orders that, and any stray `NaN`, +/// deterministically without panicking. An unknown metric name is a +/// `RegistryError::UnknownMetric`. +/// +/// The single source of truth for "best" — both [`rank_by`] (sort by it) and +/// [`optimize`] (argmax by it) call this, so the per-metric direction lives in +/// exactly one place. The metric name is resolved once via `resolve_metric`; the +/// returned closure carries the resolved `Metric`, so per-comparison work is just +/// reading each value (`metric_value`) and the key compare. +fn metric_cmp(metric: &str) -> Result Ordering, RegistryError> { + let m = resolve_metric(metric)?; + Ok(move |a: &RunReport, b: &RunReport| { + let (va, vb) = (metric_value(a, m), metric_value(b, m)); + match m { + // lower-is-better + Metric::MaxDrawdown | Metric::BiasSignFlips => va.total_cmp(&vb), + // higher-is-better + _ => vb.total_cmp(&va), } }) } @@ -193,6 +202,106 @@ pub fn optimize(family: &SweepFamily, metric: &str) -> Result bool { + matches!(m, Metric::Sqn | Metric::SqnNormalized | Metric::ExpectancyR | Metric::NetExpectancyR) +} + +fn member_trade_rs(rep: &RunReport) -> &[f64] { + rep.metrics.r.as_ref().map(|r| r.trade_rs.as_slice()).unwrap_or(&[]) +} + +/// Recompute the selected R metric from a trade-R slice (reuses the engine's +/// `r_metrics_from_rs`, the same arithmetic `summarize_r` uses). +fn member_metric_from_rs(rs: &[f64], m: Metric) -> f64 { + let rm = r_metrics_from_rs(rs); + match m { + Metric::Sqn => rm.sqn, + Metric::SqnNormalized => rm.sqn_normalized, + Metric::ExpectancyR => rm.expectancy_r, + Metric::NetExpectancyR => rm.net_expectancy_r, + _ => unreachable!("member_metric_from_rs is R-only"), + } +} + +/// The centred best-of-K null-max distribution: each member's `trade_rs` is +/// mean-subtracted (the no-edge null), then for each resample every member is +/// moving-block-resampled (in odometer order, from a per-iteration seed) and the +/// max recomputed metric across members is taken. Deterministic given `seed` (C1). +fn null_best_of_k(family: &SweepFamily, m: Metric, n_resamples: usize, block_len: usize, seed: u64) -> Vec { + let centred: Vec> = family.points.iter().map(|p| { + let rs = member_trade_rs(&p.report); + if rs.is_empty() { return Vec::new(); } + let mean = rs.iter().sum::() / rs.len() as f64; + rs.iter().map(|x| x - mean).collect() + }).collect(); + (0..n_resamples).map(|i| { + let mut rng = SplitMix64::new(seed ^ i as u64); + centred.iter().fold(f64::NEG_INFINITY, |best, rs| { + if rs.is_empty() { return best; } + let bl = block_len.clamp(1, rs.len()); + let v = member_metric_from_rs(&resample_block(rs, bl, &mut rng), m); + best.max(v) + }) + }).collect() +} + +/// Sample standard deviation of the K members' metric values (the `total_pips` +/// dispersion-floor arm). `< 2` members -> `0.0`. +fn member_sd(family: &SweepFamily, m: Metric) -> f64 { + let vals: Vec = family.points.iter().map(|p| metric_value(&p.report, m)).collect(); + let n = vals.len(); + if n < 2 { return 0.0; } + let mean = vals.iter().sum::() / n as f64; + (vals.iter().map(|v| (v - mean).powi(2)).sum::() / (n - 1) as f64).sqrt() +} + +/// `optimize`'s argmax winner PLUS its trials-deflation provenance. The returned +/// `SweepPoint` is byte-identical to `optimize(family, metric)` (additive, C23). +/// R arm: a centred moving-block reality-check (`overfit_probability` = +/// `(count(null ≥ raw) + 1)/(n + 1)`, `deflated_score` = `raw − p95(null)`). +/// `total_pips` arm: a closed-form expected-max-of-K dispersion floor, no +/// probability. Deterministic given `seed` (C1). +pub fn optimize_deflated( + family: &SweepFamily, metric: &str, n_resamples: usize, block_len: usize, seed: u64, +) -> Result<(SweepPoint, FamilySelection), RegistryError> { + let winner = optimize(family, metric)?; + let m = resolve_metric(metric)?; + let k = family.points.len(); + let raw = metric_value(&winner.report, m); + + let (deflated_score, overfit_probability) = if is_r_metric(m) { + let null_max = null_best_of_k(family, m, n_resamples, block_len, seed); + // Keep only the finite best-of-K draws. The null is *not computable* in two + // degenerate cases: zero resamples (`null_max` empty), or no member carries + // `trade_rs` (every member's centred series is empty, so each iteration's + // best-of-K folds to `NEG_INFINITY`). The latter is reachable — `trade_rs` + // is `#[serde(skip)]`, so a family loaded back from the registry has empty + // conduits even when its `r` block (hence `raw`) is finite. Filtering to + // finite values collapses both into one "no usable null" branch, instead of + // letting `p95 = NEG_INFINITY` turn `deflated_score` into `+inf`. + let usable: Vec = null_max.into_iter().filter(|x| x.is_finite()).collect(); + if usable.is_empty() { + // No reality-check is computable: floor to a degenerate-but-defined + // result — no deflation credit (`deflated_score == raw`) and a + // maximally-uncertain overfit probability (1.0), never a nonsensical inf. + (raw, Some(1.0)) + } else { + let p95 = MetricStats::from_values(&usable).p95; + let over = (usable.iter().filter(|&&x| x >= raw).count() + 1) as f64 + / (usable.len() + 1) as f64; + (raw - p95, Some(over)) + } + } else { + (raw - member_sd(family, m) * expected_max_of_normals(k), None) + }; + + Ok((winner.clone(), FamilySelection { + selection_metric: metric.to_string(), n_trials: k, raw_winner_metric: raw, + deflated_score, overfit_probability, mode: SelectionMode::Argmax, + n_resamples, block_len, seed, + })) +} + /// What can go wrong reading or ranking the registry. #[derive(Debug)] pub enum RegistryError { @@ -242,6 +351,7 @@ mod tests { window: (Timestamp(1), Timestamp(2)), seed: 0, broker: "b".to_string(), + selection: None, }, metrics: RunMetrics { total_pips, max_drawdown, bias_sign_flips: flips, r: None }, } @@ -393,6 +503,19 @@ mod tests { let _ = fs::remove_file(&path); } + /// Property: the compat read-path carries a manifest's `selection` block + /// through to the canonical `RunReport` — a line written *with* a + /// FamilySelection lifts it intact (not dropped) through `RunReportRead`'s + /// structural mirror and its `From` conversion. + #[test] + fn load_lifts_a_selection_block_through_the_compat_mirror() { + let line = r#"{"manifest":{"commit":"c","params":[],"window":[0,0],"seed":0,"broker":"b","selection":{"selection_metric":"sqn_normalized","n_trials":4,"raw_winner_metric":1.8,"deflated_score":0.2,"overfit_probability":0.06,"mode":"Argmax","n_resamples":1000,"block_len":5,"seed":42}},"metrics":{"total_pips":0.0,"max_drawdown":0.0,"bias_sign_flips":0}}"#; + let rep: RunReport = serde_json::from_str::(line).unwrap().into(); + let sel = rep.manifest.selection.expect("selection lifted"); + assert_eq!(sel.n_trials, 4); + assert_eq!(sel.mode, aura_engine::SelectionMode::Argmax); + } + #[test] fn rank_by_orders_best_first_per_metric() { let reports = vec![ @@ -547,4 +670,117 @@ mod tests { let ranked = rank_by(member_reports, "total_pips").expect("rank"); assert_eq!(ranked[0].metrics.total_pips, 3.0); // best-first within the family } + + fn member(total_pips: f64, trade_rs: Vec) -> SweepPoint { + // Every RMetrics field is derived from the R slice by `r_metrics_from_rs` + // (the same arithmetic production members carry), then the per-trade + // `trade_rs` conduit is restored: `r_metrics_from_rs` empties it + // (`trade_rs: Vec::new()`), whereas a production member is built by + // `summarize_r`, which RETAINS it — and `optimize_deflated`'s R arm + // resamples exactly that conduit, so a fixture without it cannot reach + // the R-arm path under test. + let mut r = aura_engine::r_metrics_from_rs(&trade_rs); + r.trade_rs = trade_rs; + SweepPoint { + params: vec![], + report: RunReport { + manifest: RunManifest { + commit: "c".into(), params: vec![], window: (Timestamp(0), Timestamp(0)), + seed: 0, broker: "b".into(), selection: None, + }, + metrics: RunMetrics { + total_pips, max_drawdown: 0.0, bias_sign_flips: 0, + r: Some(r), + }, + }, + } + } + + fn fixture_family_with_r() -> SweepFamily { + SweepFamily { space: vec![], points: vec![ + member(10.0, vec![1.0, -0.5, 0.8, -0.2, 0.6]), + member(30.0, vec![2.0, 1.5, -0.3, 1.0, 0.9]), // best by every metric here + member(5.0, vec![-0.5, 0.2, -0.8, 0.1, -0.3]), + ]} + } + + fn fixture_family_one_strong_edge() -> SweepFamily { + SweepFamily { space: vec![], points: vec![ + member(1.0, vec![2.0, 2.5, 1.8, 2.2, 2.1, 1.9]), // strong, consistent +R edge + member(1.0, vec![0.05, -0.1, 0.0, 0.1, -0.05, 0.02]), // ~zero-edge noise + member(1.0, vec![-0.1, 0.1, 0.0, -0.05, 0.05, 0.0]), + ]} + } + + #[test] + fn optimize_deflated_winner_is_byte_identical_to_optimize() { + let fam = fixture_family_with_r(); // helper: ≥3 members, varied sqn_normalized + trade_rs + for metric in ["total_pips", "sqn_normalized", "expectancy_r"] { + let plain = optimize(&fam, metric).unwrap(); + let (defl, _) = optimize_deflated(&fam, metric, 200, 3, 7).unwrap(); + assert_eq!(defl, plain, "deflation must not change the winner ({metric})"); + } + } + + #[test] + fn optimize_deflated_is_reproducible_from_its_seed() { + let fam = fixture_family_with_r(); + let a = optimize_deflated(&fam, "sqn_normalized", 500, 4, 42).unwrap().1; + let b = optimize_deflated(&fam, "sqn_normalized", 500, 4, 42).unwrap().1; + assert_eq!(a, b); + } + + #[test] + fn optimize_deflated_total_pips_arm_floors_without_a_probability() { + let fam = fixture_family_with_r(); + let sel = optimize_deflated(&fam, "total_pips", 100, 3, 1).unwrap().1; + assert!(sel.overfit_probability.is_none()); + assert_eq!(sel.selection_metric, "total_pips"); + assert!(sel.deflated_score <= sel.raw_winner_metric); // dispersion floor ≤ raw + } + + /// Sibling-contract parity with `r_bootstrap`, which defines `n_resamples == + /// 0` as a valid input that floors to an all-zero result. `optimize_deflated`'s + /// R arm must not panic on an empty null distribution: with zero resamples it + /// returns a degenerate-but-defined provenance — `deflated_score == raw` (the + /// p95 of an empty null is 0), `overfit_probability == 1` (the `(0 + 1)/(0 + + /// 1)` Laplace floor) — never indexing the empty `MetricStats::from_values`. + #[test] + fn optimize_deflated_zero_resamples_floors_like_r_bootstrap() { + let fam = fixture_family_with_r(); + let sel = optimize_deflated(&fam, "expectancy_r", 0, 3, 1).unwrap().1; + assert_eq!(sel.n_resamples, 0); + assert_eq!(sel.deflated_score, sel.raw_winner_metric, "p95 of an empty null is 0"); + assert_eq!(sel.overfit_probability, Some(1.0), "(0+1)/(0+1) Laplace floor"); + } + + /// Regression: a family whose members carry an `r` block but NO `trade_rs` — + /// the serde-skipped state of a family `load`ed back from the registry — must + /// NOT yield a `+inf` deflated score on the R arm with positive resamples. With + /// no member contributing a centred series, every best-of-K draw is + /// `NEG_INFINITY`; the finite-filter collapses this to the same degenerate floor + /// as zero-resamples (`deflated_score == raw`, `overfit_probability == 1.0`). + #[test] + fn optimize_deflated_no_trade_rs_floors_instead_of_infinity() { + let mut fam = fixture_family_with_r(); + for p in &mut fam.points { + if let Some(r) = p.report.metrics.r.as_mut() { + r.trade_rs.clear(); // mimic a serde-loaded member (trade_rs is #[serde(skip)]) + } + } + let sel = optimize_deflated(&fam, "expectancy_r", 500, 3, 1).unwrap().1; + assert!(sel.deflated_score.is_finite(), "deflated_score must be finite, got {}", sel.deflated_score); + assert_eq!(sel.deflated_score, sel.raw_winner_metric); + assert_eq!(sel.overfit_probability, Some(1.0)); + } + + #[test] + fn optimize_deflated_overfit_probability_is_low_for_a_clear_edge() { + // A family whose winner has a strong real R-edge over near-zero-edge peers: + // the centred best-of-K null rarely reaches the winner's raw metric. + let fam = fixture_family_one_strong_edge(); + let sel = optimize_deflated(&fam, "expectancy_r", 1000, 1, 9).unwrap().1; + assert!(sel.overfit_probability.unwrap() < 0.2, "p={:?}", sel.overfit_probability); + assert!(sel.deflated_score > 0.0); + } } diff --git a/crates/aura-registry/src/trace_store.rs b/crates/aura-registry/src/trace_store.rs index 265f90c..c01b6c2 100644 --- a/crates/aura-registry/src/trace_store.rs +++ b/crates/aura-registry/src/trace_store.rs @@ -253,6 +253,7 @@ mod tests { window: (Timestamp(1), Timestamp(5)), seed: 0, broker: "sim-optimal(pip_size=0.0001)".to_string(), + selection: None, } }