From ecc9541a5616c5e9313248a0269b896d5fb92b71 Mon Sep 17 00:00:00 2001 From: Brummel Date: Fri, 26 Jun 2026 15:00:38 +0200 Subject: [PATCH] 0076.tidy: backfill spec-promised deflation tests + guard the dispersion-floor sign MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit (architect, cycle 0076) found three tests the spec's testing strategy promised but the implement-loop did not land, plus an unguarded sign assumption: - optimize_deflated_centres_the_null_so_a_uniform_edge_is_not_called_overfit: the load-bearing protection for the mean-subtraction. A family where every member shares the same +R edge reads as NOT overfit only because the null is centred; an uncentred null would give ≈0.5. Removing the centring now flips a test (it previously passed every test — the architect's headline finding). - optimize_deflated_all_noise_family_has_high_overfit_probability: the high-p companion to the existing clear-edge low-p test. - optimize_deflated_provenance_reflects_only_the_passed_family: C2 at the registry boundary (n_trials == family size, raw == the family's own argmax), pinning that the record encodes only in-sample information. - A debug_assert in the dispersion-floor arm: it subtracts inflation, valid only for higher-is-better metrics; lower-is-better (max_drawdown/bias_sign_flips) would deflate wrong-signed. Unreached from the two shipped call sites, but optimize_deflated is public — guard the assumption rather than leave it implicit. cargo test -p aura-registry (33) + clippy green. refs #144 --- crates/aura-registry/src/lib.rs | 73 +++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/crates/aura-registry/src/lib.rs b/crates/aura-registry/src/lib.rs index 8a2e595..b6c8807 100644 --- a/crates/aura-registry/src/lib.rs +++ b/crates/aura-registry/src/lib.rs @@ -292,6 +292,16 @@ pub fn optimize_deflated( (raw - p95, Some(over)) } } else { + // The dispersion floor SUBTRACTS the expected-max inflation, so it is valid + // only for a higher-is-better metric (a larger value is the better one the + // search inflated). A lower-is-better metric would deflate wrong-signed; the + // two shipped call sites only pass `total_pips`, so this is unreached, but + // `optimize_deflated` is public — guard the assumption rather than leave it + // implicit. + debug_assert!( + !matches!(m, Metric::MaxDrawdown | Metric::BiasSignFlips), + "dispersion-floor arm is for higher-is-better metrics only", + ); (raw - member_sd(family, m) * expected_max_of_normals(k), None) }; @@ -783,4 +793,67 @@ mod tests { assert!(sel.overfit_probability.unwrap() < 0.2, "p={:?}", sel.overfit_probability); assert!(sel.deflated_score > 0.0); } + + /// Every member shares the SAME strong +R edge. An *uncentred* null (each + /// member's own edged trades resampled) would put the best-of-K ≈ the winner ⇒ + /// `overfit_probability ≈ 0.5`. The centred null removes each member's mean, so + /// the winner's real edge stands clear of a zero-edge null ⇒ a low probability. + /// This is the discriminator that the mean-subtraction (the statistic's whole + /// point) is load-bearing — removing the centring would flip this assertion. + fn fixture_family_uniform_edge() -> SweepFamily { + SweepFamily { space: vec![], points: vec![ + member(1.0, vec![1.8, 2.2, 1.9, 2.1, 2.0, 1.7]), + member(1.0, vec![2.1, 1.9, 2.0, 2.2, 1.8, 2.0]), + member(1.0, vec![1.9, 2.0, 2.1, 1.8, 2.2, 1.9]), + ]} + } + + #[test] + fn optimize_deflated_centres_the_null_so_a_uniform_edge_is_not_called_overfit() { + let fam = fixture_family_uniform_edge(); + let sel = optimize_deflated(&fam, "expectancy_r", 1000, 1, 5).unwrap().1; + assert!( + sel.overfit_probability.unwrap() < 0.2, + "uniform real edge must read as NOT overfit under the centred null; an \ + uncentred null would give ≈0.5. p={:?}", + sel.overfit_probability + ); + } + + /// No member has a real edge — all are small zero-mean noise. The winner is the + /// best-of-K by luck, and the centred best-of-K null reaches its tiny raw edge + /// routinely ⇒ a HIGH overfit probability (the companion to the clear-edge + /// low-p case: the statistic must distinguish luck from edge). + fn fixture_family_all_noise() -> SweepFamily { + SweepFamily { space: vec![], points: vec![ + member(1.0, vec![0.3, -0.4, 0.2, -0.1, 0.1, -0.2, 0.15, -0.05]), + member(1.0, vec![-0.2, 0.3, -0.3, 0.1, 0.0, 0.2, -0.1, 0.05]), + member(1.0, vec![0.1, -0.1, 0.25, -0.2, -0.05, 0.15, 0.0, -0.1]), + ]} + } + + #[test] + fn optimize_deflated_all_noise_family_has_high_overfit_probability() { + let fam = fixture_family_all_noise(); + let sel = optimize_deflated(&fam, "expectancy_r", 1000, 1, 5).unwrap().1; + assert!( + sel.overfit_probability.unwrap() > 0.3, + "an all-noise family's lucky winner must read as overfit-prone, p={:?}", + sel.overfit_probability + ); + } + + /// C2 at the registry boundary: `optimize_deflated` has no access to any + /// out-of-sample report — its provenance is a pure function of the in-sample + /// family the caller passes. `n_trials` is exactly that family's size and + /// `raw_winner_metric` is that family's own argmax value, so the record cannot + /// encode out-of-sample information. + #[test] + fn optimize_deflated_provenance_reflects_only_the_passed_family() { + let fam = fixture_family_with_r(); + let (winner, sel) = optimize_deflated(&fam, "sqn_normalized", 200, 3, 1).unwrap(); + assert_eq!(sel.n_trials, fam.points.len()); + let m = resolve_metric("sqn_normalized").unwrap(); + assert_eq!(sel.raw_winner_metric, metric_value(&winner.report, m)); + } }