0076.tidy: backfill spec-promised deflation tests + guard the dispersion-floor sign
Audit (architect, cycle 0076) found three tests the spec's testing strategy promised but the implement-loop did not land, plus an unguarded sign assumption: - optimize_deflated_centres_the_null_so_a_uniform_edge_is_not_called_overfit: the load-bearing protection for the mean-subtraction. A family where every member shares the same +R edge reads as NOT overfit only because the null is centred; an uncentred null would give ≈0.5. Removing the centring now flips a test (it previously passed every test — the architect's headline finding). - optimize_deflated_all_noise_family_has_high_overfit_probability: the high-p companion to the existing clear-edge low-p test. - optimize_deflated_provenance_reflects_only_the_passed_family: C2 at the registry boundary (n_trials == family size, raw == the family's own argmax), pinning that the record encodes only in-sample information. - A debug_assert in the dispersion-floor arm: it subtracts inflation, valid only for higher-is-better metrics; lower-is-better (max_drawdown/bias_sign_flips) would deflate wrong-signed. Unreached from the two shipped call sites, but optimize_deflated is public — guard the assumption rather than leave it implicit. cargo test -p aura-registry (33) + clippy green. refs #144
This commit is contained in:
@@ -292,6 +292,16 @@ pub fn optimize_deflated(
|
|||||||
(raw - p95, Some(over))
|
(raw - p95, Some(over))
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
// The dispersion floor SUBTRACTS the expected-max inflation, so it is valid
|
||||||
|
// only for a higher-is-better metric (a larger value is the better one the
|
||||||
|
// search inflated). A lower-is-better metric would deflate wrong-signed; the
|
||||||
|
// two shipped call sites only pass `total_pips`, so this is unreached, but
|
||||||
|
// `optimize_deflated` is public — guard the assumption rather than leave it
|
||||||
|
// implicit.
|
||||||
|
debug_assert!(
|
||||||
|
!matches!(m, Metric::MaxDrawdown | Metric::BiasSignFlips),
|
||||||
|
"dispersion-floor arm is for higher-is-better metrics only",
|
||||||
|
);
|
||||||
(raw - member_sd(family, m) * expected_max_of_normals(k), None)
|
(raw - member_sd(family, m) * expected_max_of_normals(k), None)
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -783,4 +793,67 @@ mod tests {
|
|||||||
assert!(sel.overfit_probability.unwrap() < 0.2, "p={:?}", sel.overfit_probability);
|
assert!(sel.overfit_probability.unwrap() < 0.2, "p={:?}", sel.overfit_probability);
|
||||||
assert!(sel.deflated_score > 0.0);
|
assert!(sel.deflated_score > 0.0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Every member shares the SAME strong +R edge. An *uncentred* null (each
|
||||||
|
/// member's own edged trades resampled) would put the best-of-K ≈ the winner ⇒
|
||||||
|
/// `overfit_probability ≈ 0.5`. The centred null removes each member's mean, so
|
||||||
|
/// the winner's real edge stands clear of a zero-edge null ⇒ a low probability.
|
||||||
|
/// This is the discriminator that the mean-subtraction (the statistic's whole
|
||||||
|
/// point) is load-bearing — removing the centring would flip this assertion.
|
||||||
|
fn fixture_family_uniform_edge() -> SweepFamily {
|
||||||
|
SweepFamily { space: vec![], points: vec![
|
||||||
|
member(1.0, vec![1.8, 2.2, 1.9, 2.1, 2.0, 1.7]),
|
||||||
|
member(1.0, vec![2.1, 1.9, 2.0, 2.2, 1.8, 2.0]),
|
||||||
|
member(1.0, vec![1.9, 2.0, 2.1, 1.8, 2.2, 1.9]),
|
||||||
|
]}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn optimize_deflated_centres_the_null_so_a_uniform_edge_is_not_called_overfit() {
|
||||||
|
let fam = fixture_family_uniform_edge();
|
||||||
|
let sel = optimize_deflated(&fam, "expectancy_r", 1000, 1, 5).unwrap().1;
|
||||||
|
assert!(
|
||||||
|
sel.overfit_probability.unwrap() < 0.2,
|
||||||
|
"uniform real edge must read as NOT overfit under the centred null; an \
|
||||||
|
uncentred null would give ≈0.5. p={:?}",
|
||||||
|
sel.overfit_probability
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// No member has a real edge — all are small zero-mean noise. The winner is the
|
||||||
|
/// best-of-K by luck, and the centred best-of-K null reaches its tiny raw edge
|
||||||
|
/// routinely ⇒ a HIGH overfit probability (the companion to the clear-edge
|
||||||
|
/// low-p case: the statistic must distinguish luck from edge).
|
||||||
|
fn fixture_family_all_noise() -> SweepFamily {
|
||||||
|
SweepFamily { space: vec![], points: vec![
|
||||||
|
member(1.0, vec![0.3, -0.4, 0.2, -0.1, 0.1, -0.2, 0.15, -0.05]),
|
||||||
|
member(1.0, vec![-0.2, 0.3, -0.3, 0.1, 0.0, 0.2, -0.1, 0.05]),
|
||||||
|
member(1.0, vec![0.1, -0.1, 0.25, -0.2, -0.05, 0.15, 0.0, -0.1]),
|
||||||
|
]}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn optimize_deflated_all_noise_family_has_high_overfit_probability() {
|
||||||
|
let fam = fixture_family_all_noise();
|
||||||
|
let sel = optimize_deflated(&fam, "expectancy_r", 1000, 1, 5).unwrap().1;
|
||||||
|
assert!(
|
||||||
|
sel.overfit_probability.unwrap() > 0.3,
|
||||||
|
"an all-noise family's lucky winner must read as overfit-prone, p={:?}",
|
||||||
|
sel.overfit_probability
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// C2 at the registry boundary: `optimize_deflated` has no access to any
|
||||||
|
/// out-of-sample report — its provenance is a pure function of the in-sample
|
||||||
|
/// family the caller passes. `n_trials` is exactly that family's size and
|
||||||
|
/// `raw_winner_metric` is that family's own argmax value, so the record cannot
|
||||||
|
/// encode out-of-sample information.
|
||||||
|
#[test]
|
||||||
|
fn optimize_deflated_provenance_reflects_only_the_passed_family() {
|
||||||
|
let fam = fixture_family_with_r();
|
||||||
|
let (winner, sel) = optimize_deflated(&fam, "sqn_normalized", 200, 3, 1).unwrap();
|
||||||
|
assert_eq!(sel.n_trials, fam.points.len());
|
||||||
|
let m = resolve_metric("sqn_normalized").unwrap();
|
||||||
|
assert_eq!(sel.raw_winner_metric, metric_value(&winner.report, m));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user