feat(0076): deflate the sweep winner's metric for the number of trials
Adds optimize_deflated beside optimize in aura-registry: the in-sample walk-forward selection now records, on each OOS winner's manifest, how much the winner's metric is inflated by the size of the search it won — a deflated score and (R arm) an overfit probability — without changing which member wins (additive, C23; a regression test pins optimize_deflated's winner == optimize's). - R arm (sqn_normalized/expectancy_r/...): a centred moving-block reality-check. Each member's per-trade R series is mean-subtracted to impose the no-edge null, resampled via the r_bootstrap kernel (extracted to a shared resample_block, byte-identical), and the best-of-K null-max gives overfit_probability = (count(null >= raw)+1)/(n+1) and deflated_score = raw - p95(null). Deterministic given the seed (C1). - total_pips arm: a closed-form expected-max-of-K dispersion floor (inv_norm_cdf + expected_max_of_normals), no probability. The record (FamilySelection on RunManifest) rides the proven serde(default, skip_serializing_if) widening + a compat.rs mirror, so legacy runs.jsonl/families.jsonl lines load unchanged and unstamped sweep/mc/standalone manifests stay byte-identical (C14/C18). The SelectionMode enum reserves a Plateau slot for #145. CLI: walk-forward stamps both arms; runs family <id> rank surfaces a human-readable deflated/overfit line. Quality review (re-dispatched after the implement-loop's quality gate exhausted) caught a real defect: on the R arm with positive resamples but no member carrying trade_rs (a family loaded from disk — trade_rs is serde-skipped), the null was all-NEG_INFINITY, turning deflated_score into +inf. Fixed by collapsing "no usable null" (zero resamples OR no trade_rs) into one degenerate floor (deflated == raw, overfit == 1.0); pinned by optimize_deflated_no_trade_rs_floors_instead_of_infinity. Verified: cargo test --workspace and cargo clippy --workspace --all-targets -D warnings both green. Frictionless Stage-1 R; n_eff advisory and reload-time recompute deferred (spec 0076 Out of scope). closes #144
This commit is contained in:
@@ -24,7 +24,7 @@ use aura_engine::{
|
||||
WalkForwardResult, WindowBounds, WindowRoller, WindowRun,
|
||||
};
|
||||
use aura_registry::{
|
||||
group_families, mc_member_reports, optimize, rank_by, sweep_member_reports,
|
||||
group_families, mc_member_reports, optimize_deflated, rank_by, sweep_member_reports,
|
||||
walkforward_member_reports, FamilyKind, FamilyMember, NameKind, Registry, RunTraces, TraceStore,
|
||||
WriteKind,
|
||||
};
|
||||
@@ -52,6 +52,13 @@ const WF_REAL_IS_NS: i64 = 90 * WF_DAY_NS;
|
||||
const WF_REAL_OOS_NS: i64 = 30 * WF_DAY_NS;
|
||||
const WF_REAL_STEP_NS: i64 = 30 * WF_DAY_NS;
|
||||
|
||||
/// Trials-deflation resampling budget for walk-forward winner selection. Recorded
|
||||
/// on each winner's manifest (so `overfit_probability` is reproducible by re-run);
|
||||
/// CLI flags for these are a deferred refinement.
|
||||
const DEFLATION_N_RESAMPLES: usize = 1000;
|
||||
const DEFLATION_BLOCK_LEN: usize = 5;
|
||||
const DEFLATION_SEED: u64 = 0xDEF1_A7ED;
|
||||
|
||||
/// The built-in synthetic price stream: rises through t=4 then reverses, so the
|
||||
/// demo trace carries one exposure sign flip and a real drawdown (C22 populated
|
||||
/// trace). Deterministic and fixed (C1).
|
||||
@@ -166,6 +173,7 @@ fn sim_optimal_manifest(
|
||||
window,
|
||||
seed,
|
||||
broker: format!("sim-optimal(pip_size={pip_size})"),
|
||||
selection: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1792,8 +1800,11 @@ fn walkforward_family(
|
||||
let space = sample_blueprint_with_sinks(data.pip_size()).0.param_space();
|
||||
walk_forward(roller, space, |w: WindowBounds| {
|
||||
let is_family = sweep_over(w.is.0, w.is.1, data);
|
||||
let best = optimize(&is_family, "total_pips").expect("total_pips is a known metric");
|
||||
let (oos_equity, oos_report) = run_oos(&best.params, w.oos.0, w.oos.1, trace, data);
|
||||
let (best, selection) = optimize_deflated(
|
||||
&is_family, "total_pips", DEFLATION_N_RESAMPLES, DEFLATION_BLOCK_LEN, DEFLATION_SEED,
|
||||
).expect("total_pips is a known metric");
|
||||
let (oos_equity, mut oos_report) = run_oos(&best.params, w.oos.0, w.oos.1, trace, data);
|
||||
oos_report.manifest.selection = Some(selection);
|
||||
WindowRun {
|
||||
// The tag-free sweep winner is the chosen point; its kinds live on
|
||||
// WalkForwardResult.space (computed once above from the same blueprint).
|
||||
@@ -1807,8 +1818,11 @@ fn walkforward_family(
|
||||
let space = stage1_r_space();
|
||||
walk_forward(roller, space, |w: WindowBounds| {
|
||||
let is_family = stage1_r_sweep_over(w.is.0, w.is.1, data, grid);
|
||||
let best = optimize(&is_family, "sqn_normalized").expect("sqn_normalized is a known metric");
|
||||
let (oos_equity, oos_report) = run_oos_r(&best.params, w.oos.0, w.oos.1, trace, data);
|
||||
let (best, selection) = optimize_deflated(
|
||||
&is_family, "sqn_normalized", DEFLATION_N_RESAMPLES, DEFLATION_BLOCK_LEN, DEFLATION_SEED,
|
||||
).expect("sqn_normalized is a known metric");
|
||||
let (oos_equity, mut oos_report) = run_oos_r(&best.params, w.oos.0, w.oos.1, trace, data);
|
||||
oos_report.manifest.selection = Some(selection);
|
||||
WindowRun { chosen_params: best.params, oos_equity, oos_report }
|
||||
})
|
||||
}
|
||||
@@ -2200,6 +2214,12 @@ fn runs_family(id: &str, rank: Option<&str>) {
|
||||
};
|
||||
for report in &ordered {
|
||||
println!("{}", report.to_json());
|
||||
if let Some(sel) = &report.manifest.selection {
|
||||
match sel.overfit_probability {
|
||||
Some(p) => println!(" deflated={:.4} P(overfit)={:.4}", sel.deflated_score, p),
|
||||
None => println!(" deflated={:.4}", sel.deflated_score),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2349,6 +2349,44 @@ fn walkforward_strategy_stage1_r_reports_oos_r() {
|
||||
assert_eq!(out, out2, "stage1-r walkforward is deterministic");
|
||||
}
|
||||
|
||||
/// Property: a stage1-r walk-forward stamps each OOS winner's manifest with the
|
||||
/// trials-deflation provenance (#144), and `runs family <id> rank` surfaces it as a
|
||||
/// human-readable `deflated=… P(overfit)=…` line beside each member's JSON. This is
|
||||
/// the end-to-end witness that the selection record reaches disk (C18) and the
|
||||
/// display path renders it; sweep/mc families stay unstamped (verified elsewhere).
|
||||
#[test]
|
||||
fn runs_family_rank_shows_deflated_line() {
|
||||
let cwd = temp_cwd("runs-deflated");
|
||||
let wf = Command::new(BIN)
|
||||
.args(["walkforward", "--strategy", "stage1-r", "--name", "wf-defl"])
|
||||
.current_dir(&cwd)
|
||||
.output()
|
||||
.expect("spawn walkforward --strategy stage1-r --name wf-defl");
|
||||
assert!(
|
||||
wf.status.success(),
|
||||
"walkforward exit: {:?}; stderr: {}",
|
||||
wf.status,
|
||||
String::from_utf8_lossy(&wf.stderr)
|
||||
);
|
||||
|
||||
let rank = Command::new(BIN)
|
||||
.args(["runs", "family", "wf-defl-0", "rank", "sqn_normalized"])
|
||||
.current_dir(&cwd)
|
||||
.output()
|
||||
.expect("spawn runs family wf-defl-0 rank sqn_normalized");
|
||||
assert!(rank.status.success(), "rank exit: {:?}", rank.status);
|
||||
let rank_out = String::from_utf8(rank.stdout).expect("utf-8");
|
||||
let _ = std::fs::remove_dir_all(&cwd);
|
||||
|
||||
assert!(rank_out.contains("deflated="), "rank output missing deflated=: {rank_out:?}");
|
||||
assert!(rank_out.contains("P(overfit)="), "rank output missing P(overfit)=: {rank_out:?}");
|
||||
// each stamped member's JSON also carries the selection block (selection_metric).
|
||||
assert!(
|
||||
rank_out.contains("\"selection_metric\":\"sqn_normalized\""),
|
||||
"selection block present: {rank_out:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Property: the bare `aura walkforward` (default SMA-cross) summary is preserved
|
||||
/// byte-for-byte by the new `--strategy`/grid plumbing — it still carries exactly
|
||||
/// `windows`, `stitched_total_pips`, `param_stability` and NOT the stage1-r-only
|
||||
|
||||
@@ -932,6 +932,7 @@ mod tests {
|
||||
window: (Timestamp(0), Timestamp(0)),
|
||||
seed: 0,
|
||||
broker: "test".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics: summarize(&equity, &exposure),
|
||||
}
|
||||
|
||||
@@ -140,15 +140,15 @@ pub fn window_of(sources: &[Box<dyn Source>]) -> Option<(Timestamp, Timestamp)>
|
||||
/// completely determines the sequence; no external entropy, no global state.
|
||||
/// Bit-stable across toolchains and crate versions — the property C1 needs for
|
||||
/// seed-as-input reproducibility (C12).
|
||||
pub(crate) struct SplitMix64 {
|
||||
pub struct SplitMix64 {
|
||||
state: u64,
|
||||
}
|
||||
|
||||
impl SplitMix64 {
|
||||
pub(crate) fn new(seed: u64) -> Self {
|
||||
pub fn new(seed: u64) -> Self {
|
||||
Self { state: seed }
|
||||
}
|
||||
pub(crate) fn next_u64(&mut self) -> u64 {
|
||||
pub fn next_u64(&mut self) -> u64 {
|
||||
self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = self.state;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
|
||||
@@ -58,18 +58,21 @@ pub use blueprint::{
|
||||
pub use builder::{BuildError, GraphBuilder, InPort, NodeHandle, OutPort, RoleHandle};
|
||||
pub use graph_model::model_to_json;
|
||||
pub use harness::{
|
||||
window_of, BootstrapError, Edge, FlatGraph, Harness, Source, SourceSpec, SyntheticSpec, Target,
|
||||
VecSource,
|
||||
window_of, BootstrapError, Edge, FlatGraph, Harness, Source, SourceSpec, SplitMix64,
|
||||
SyntheticSpec, Target, VecSource,
|
||||
};
|
||||
pub use report::{
|
||||
derive_position_events, f64_field, join_on_ts, r_metrics_from_rs, summarize, summarize_r,
|
||||
ColumnarTrace, JoinedRow, PositionAction, PositionEvent, RMetrics, RunManifest, RunMetrics,
|
||||
RunReport,
|
||||
derive_position_events, expected_max_of_normals, f64_field, inv_norm_cdf, join_on_ts,
|
||||
r_metrics_from_rs, summarize, summarize_r, ColumnarTrace, FamilySelection, JoinedRow,
|
||||
PositionAction, PositionEvent, RMetrics, RunManifest, RunMetrics, RunReport, SelectionMode,
|
||||
};
|
||||
pub use sweep::{
|
||||
sweep, GridSpace, ParamRange, RandomSpace, Space, SweepError, SweepFamily, SweepPoint,
|
||||
};
|
||||
pub use mc::{monte_carlo, r_bootstrap, McAggregate, McDraw, McFamily, MetricStats, RBootstrap};
|
||||
pub use mc::{
|
||||
monte_carlo, r_bootstrap, resample_block, McAggregate, McDraw, McFamily, MetricStats,
|
||||
RBootstrap,
|
||||
};
|
||||
pub use walkforward::{
|
||||
param_stability, walk_forward, RollMode, WalkForwardError, WalkForwardResult,
|
||||
WindowBounds, WindowOutcome, WindowRoller, WindowRun,
|
||||
|
||||
@@ -166,6 +166,26 @@ pub struct RBootstrap {
|
||||
pub n_resamples: usize,
|
||||
}
|
||||
|
||||
/// One moving-block resample of `rs` to length `rs.len()` (non-circular; the final
|
||||
/// block is truncated so the resample has exactly `n` values). `block_len` must be
|
||||
/// pre-clamped to `[1, rs.len()]` by the caller. Pure given the `rng` state — the
|
||||
/// shared kernel behind `r_bootstrap` and the trials-deflation reality-check (C1).
|
||||
pub fn resample_block(rs: &[f64], block_len: usize, rng: &mut SplitMix64) -> Vec<f64> {
|
||||
let n = rs.len();
|
||||
debug_assert!(
|
||||
(1..=n).contains(&block_len),
|
||||
"resample_block requires block_len in [1, rs.len()] (caller pre-clamps); \
|
||||
got block_len={block_len}, rs.len()={n}",
|
||||
);
|
||||
let mut sample: Vec<f64> = Vec::with_capacity(n);
|
||||
while sample.len() < n {
|
||||
let start = (rng.next_u64() % (n - block_len + 1) as u64) as usize;
|
||||
let take = block_len.min(n - sample.len());
|
||||
sample.extend_from_slice(&rs[start..start + take]);
|
||||
}
|
||||
sample
|
||||
}
|
||||
|
||||
/// Moving-block bootstrap of `rs` (non-circular; the final block of each resample is
|
||||
/// truncated so the resample has exactly `n` values). `block_len` is clamped to
|
||||
/// `[1, n]`. Empty `rs` or zero resamples -> an all-zero `RBootstrap`. Pure given
|
||||
@@ -185,12 +205,7 @@ pub fn r_bootstrap(rs: &[f64], n_resamples: usize, block_len: usize, seed: u64)
|
||||
let mut rng = SplitMix64::new(seed);
|
||||
let mut means: Vec<f64> = Vec::with_capacity(n_resamples);
|
||||
for _ in 0..n_resamples {
|
||||
let mut sample: Vec<f64> = Vec::with_capacity(n);
|
||||
while sample.len() < n {
|
||||
let start = (rng.next_u64() % (n - block_len + 1) as u64) as usize;
|
||||
let take = block_len.min(n - sample.len());
|
||||
sample.extend_from_slice(&rs[start..start + take]);
|
||||
}
|
||||
let sample = resample_block(rs, block_len, &mut rng);
|
||||
means.push(sample.iter().sum::<f64>() / n as f64);
|
||||
}
|
||||
let e_r = MetricStats::from_values(&means);
|
||||
@@ -227,6 +242,7 @@ mod tests {
|
||||
window,
|
||||
seed,
|
||||
broker: "test".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics: summarize(&equity, &exposure),
|
||||
}
|
||||
@@ -250,6 +266,7 @@ mod tests {
|
||||
window: (Timestamp(0), Timestamp(0)),
|
||||
seed: i as u64,
|
||||
broker: "t".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics: RunMetrics {
|
||||
total_pips: v,
|
||||
@@ -359,6 +376,42 @@ mod tests {
|
||||
assert_eq!(back, s);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resample_block_returns_n_values_from_contiguous_runs() {
|
||||
// The extracted kernel's standalone contract (now a public entry point):
|
||||
// given a pre-clamped block_len in [1, n], it yields exactly `n` values, each
|
||||
// appearing in `rs`, arranged as contiguous runs of `rs` (the final run
|
||||
// truncated to reach exactly n). Distinct powers of ten make membership a sum
|
||||
// check. n=5, block_len=2 -> runs of length 2, 2, 1.
|
||||
let rs = [1.0, 10.0, 100.0, 1000.0, 10000.0];
|
||||
let n = rs.len();
|
||||
let block_len = 2;
|
||||
let mut rng = SplitMix64::new(7);
|
||||
let sample = resample_block(&rs, block_len, &mut rng);
|
||||
assert_eq!(sample.len(), n, "resample has exactly n values");
|
||||
assert!(
|
||||
sample.iter().all(|v| rs.contains(v)),
|
||||
"every resampled value is drawn from rs",
|
||||
);
|
||||
// Reconstruct the contiguous-run structure: walk the sample in blocks of
|
||||
// `block_len` (last truncated) and assert each block is a contiguous slice of rs.
|
||||
let mut filled = 0usize;
|
||||
while filled < n {
|
||||
let take = block_len.min(n - filled);
|
||||
let run = &sample[filled..filled + take];
|
||||
let start = rs
|
||||
.iter()
|
||||
.position(|v| v == &run[0])
|
||||
.expect("run head is in rs");
|
||||
assert_eq!(
|
||||
run,
|
||||
&rs[start..start + take],
|
||||
"each run is a contiguous slice of rs",
|
||||
);
|
||||
filled += take;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn r_bootstrap_empty_series_is_all_zero() {
|
||||
let b = r_bootstrap(&[], 100, 1, 7);
|
||||
|
||||
@@ -38,6 +38,33 @@ pub struct RunMetrics {
|
||||
pub r: Option<RMetrics>,
|
||||
}
|
||||
|
||||
/// Which selection objective produced the record (additive provenance, C23).
|
||||
/// `Argmax` is the bare-best pick (cycle 0076). The enum is deliberately left
|
||||
/// open: cycle 0145 adds a `Plateau*` variant on this same field, so peak-vs-
|
||||
/// plateau runs stay distinguishable.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
pub enum SelectionMode {
|
||||
Argmax,
|
||||
}
|
||||
|
||||
/// Selection-provenance for a sweep winner: how its metric was deflated for the
|
||||
/// number of configurations it beat. Additive — recorded, never re-ranking (C23).
|
||||
/// `overfit_probability` is the empirical data-snooping p-value (R arm only);
|
||||
/// `None` on the `total_pips` dispersion-floor arm.
|
||||
#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct FamilySelection {
|
||||
pub selection_metric: String,
|
||||
pub n_trials: usize,
|
||||
pub raw_winner_metric: f64,
|
||||
pub deflated_score: f64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub overfit_probability: Option<f64>,
|
||||
pub mode: SelectionMode,
|
||||
pub n_resamples: usize,
|
||||
pub block_len: usize,
|
||||
pub seed: u64,
|
||||
}
|
||||
|
||||
/// R-based signal-quality metrics (Stage-1), reduced from a `PositionManagement` dense
|
||||
/// record stream by [`summarize_r`]. Account- and instrument-agnostic (pure R). Carries
|
||||
/// the enriched dispersion/churn fields (SQN, conviction terciles, net-of-cost).
|
||||
@@ -431,6 +458,11 @@ pub struct RunManifest {
|
||||
pub seed: u64,
|
||||
/// The broker profile label, e.g. `"sim-optimal(pip_size=0.0001)"`.
|
||||
pub broker: String,
|
||||
/// Selection provenance, present only on a sweep/walk-forward winner; a
|
||||
/// standalone run and every pre-0076 line read back as `None`. One-directional
|
||||
/// serde widening (C14/C18), identical idiom to `RunMetrics.r`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selection: Option<FamilySelection>,
|
||||
}
|
||||
|
||||
/// A run's full structured result: the descriptor plus the metrics it
|
||||
@@ -633,6 +665,56 @@ fn scalar_to_f64(s: Scalar) -> f64 {
|
||||
}
|
||||
}
|
||||
|
||||
/// Inverse standard-normal CDF (quantile function), Acklam's rational
|
||||
/// approximation — absolute error < ~1.2e-9 over `p ∈ (0,1)`. Pure (C1).
|
||||
/// Callers pass strictly-interior `p` (`k >= 2` keeps the argument off the
|
||||
/// boundaries); the boundary branches return ±inf as a defined limit.
|
||||
// Acklam's published coefficients verbatim; some carry more decimals than an
|
||||
// f64 holds (clippy::excessive_precision) — kept literal as reference constants.
|
||||
#[allow(clippy::excessive_precision)]
|
||||
pub fn inv_norm_cdf(p: f64) -> f64 {
|
||||
const A: [f64; 6] = [-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02,
|
||||
1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00];
|
||||
const B: [f64; 5] = [-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02,
|
||||
6.680131188771972e+01, -1.328068155288572e+01];
|
||||
const C: [f64; 6] = [-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00,
|
||||
-2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00];
|
||||
const D: [f64; 4] = [7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00,
|
||||
3.754408661907416e+00];
|
||||
const P_LOW: f64 = 0.02425;
|
||||
if p <= 0.0 { return f64::NEG_INFINITY; }
|
||||
if p >= 1.0 { return f64::INFINITY; }
|
||||
if p < P_LOW {
|
||||
let q = (-2.0 * p.ln()).sqrt();
|
||||
(((((C[0]*q + C[1])*q + C[2])*q + C[3])*q + C[4])*q + C[5])
|
||||
/ ((((D[0]*q + D[1])*q + D[2])*q + D[3])*q + 1.0)
|
||||
} else if p <= 1.0 - P_LOW {
|
||||
let q = p - 0.5;
|
||||
let r = q * q;
|
||||
(((((A[0]*r + A[1])*r + A[2])*r + A[3])*r + A[4])*r + A[5]) * q
|
||||
/ (((((B[0]*r + B[1])*r + B[2])*r + B[3])*r + B[4])*r + 1.0)
|
||||
} else {
|
||||
let q = (-2.0 * (1.0 - p).ln()).sqrt();
|
||||
-(((((C[0]*q + C[1])*q + C[2])*q + C[3])*q + C[4])*q + C[5])
|
||||
/ ((((D[0]*q + D[1])*q + D[2])*q + D[3])*q + 1.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Expected maximum of `k` i.i.d. standard normals — the hurdle a search of `k`
|
||||
/// configurations clears by chance alone. `k <= 1` -> `0.0` (a single trial has no
|
||||
/// multiple-comparison inflation, and guards the `Φ⁻¹(1 - 1/k) -> Φ⁻¹(0)`
|
||||
/// divergence). For `k >= 2`:
|
||||
/// `(1 - γ)·Φ⁻¹(1 - 1/k) + γ·Φ⁻¹(1 - 1/(k·e))`, `γ = 0.5772156649…`. Pure (C1).
|
||||
pub fn expected_max_of_normals(k: usize) -> f64 {
|
||||
if k <= 1 {
|
||||
return 0.0;
|
||||
}
|
||||
const GAMMA: f64 = 0.577_215_664_901_532_9;
|
||||
let kf = k as f64;
|
||||
(1.0 - GAMMA) * inv_norm_cdf(1.0 - 1.0 / kf)
|
||||
+ GAMMA * inv_norm_cdf(1.0 - 1.0 / (kf * std::f64::consts::E))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -641,6 +723,57 @@ mod tests {
|
||||
use aura_std::{Bias, Recorder, SimBroker, Sma, Sub};
|
||||
use std::sync::mpsc;
|
||||
|
||||
#[test]
|
||||
fn inv_norm_cdf_matches_known_quantiles() {
|
||||
assert!((inv_norm_cdf(0.975) - 1.959964).abs() < 1e-3);
|
||||
assert!(inv_norm_cdf(0.5).abs() < 1e-9);
|
||||
assert!((inv_norm_cdf(0.1) + inv_norm_cdf(0.9)).abs() < 1e-9); // symmetry
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expected_max_of_normals_is_zero_at_one_and_monotone() {
|
||||
assert_eq!(expected_max_of_normals(1), 0.0);
|
||||
assert_eq!(expected_max_of_normals(0), 0.0);
|
||||
let (e2, e4, e10, e100) = (
|
||||
expected_max_of_normals(2), expected_max_of_normals(4),
|
||||
expected_max_of_normals(10), expected_max_of_normals(100),
|
||||
);
|
||||
assert!(e2 < e4 && e4 < e10 && e10 < e100); // strictly increasing
|
||||
assert!((1.0..1.1).contains(&e4)); // ~1.05 (not the √(2 ln 4)=1.66 asymptote)
|
||||
assert!((2.4..2.7).contains(&e100)); // ~2.53
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runmanifest_without_selection_field_deserialises_to_none() {
|
||||
let json = r#"{"commit":"c","params":[],"window":[0,0],"seed":0,"broker":"b"}"#;
|
||||
let m: RunManifest = serde_json::from_str(json).expect("legacy manifest loads");
|
||||
assert!(m.selection.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runmanifest_without_selection_serialises_without_the_key() {
|
||||
let m = RunManifest {
|
||||
commit: "c".into(), params: vec![], window: (Timestamp(0), Timestamp(0)),
|
||||
seed: 0, broker: "b".into(), selection: None,
|
||||
};
|
||||
assert!(!serde_json::to_string(&m).unwrap().contains("selection"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn family_selection_round_trips_on_the_manifest() {
|
||||
let sel = FamilySelection {
|
||||
selection_metric: "sqn_normalized".into(), n_trials: 4, raw_winner_metric: 1.83,
|
||||
deflated_score: 0.21, overfit_probability: Some(0.06), mode: SelectionMode::Argmax,
|
||||
n_resamples: 1000, block_len: 5, seed: 42,
|
||||
};
|
||||
let m = RunManifest {
|
||||
commit: "c".into(), params: vec![], window: (Timestamp(0), Timestamp(0)),
|
||||
seed: 0, broker: "b".into(), selection: Some(sel.clone()),
|
||||
};
|
||||
let back: RunManifest = serde_json::from_str(&serde_json::to_string(&m).unwrap()).unwrap();
|
||||
assert_eq!(back.selection, Some(sel));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn position_action_round_trips_through_i64() {
|
||||
for a in [PositionAction::Buy, PositionAction::Sell, PositionAction::Close] {
|
||||
@@ -785,6 +918,7 @@ mod tests {
|
||||
window: (Timestamp(1), Timestamp(5)),
|
||||
seed: 0,
|
||||
broker: "sim-optimal(pip_size=0.0001)".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics,
|
||||
}
|
||||
@@ -1160,6 +1294,7 @@ mod tests {
|
||||
window: (Timestamp(1), Timestamp(6)),
|
||||
seed: 0,
|
||||
broker: "sim-optimal(pip_size=1.0)".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics: RunMetrics {
|
||||
total_pips: 12.0,
|
||||
@@ -1188,6 +1323,7 @@ mod tests {
|
||||
window: (Timestamp(1), Timestamp(6)),
|
||||
seed: 0,
|
||||
broker: "sim-optimal(pip_size=1.0)".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics: RunMetrics { total_pips: 12.0, max_drawdown: 1.0, bias_sign_flips: 1, r: None },
|
||||
};
|
||||
@@ -1208,6 +1344,7 @@ mod tests {
|
||||
window: (Timestamp(1), Timestamp(6)),
|
||||
seed: 0,
|
||||
broker: "sim-optimal(pip_size=1.0)".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics: RunMetrics { total_pips: 12.0, max_drawdown: 1.0, bias_sign_flips: 1, r: None },
|
||||
};
|
||||
|
||||
@@ -646,6 +646,7 @@ mod tests {
|
||||
window: (Timestamp(0), Timestamp(0)),
|
||||
seed: 0,
|
||||
broker: "test".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics: summarize(&equity, &exposure),
|
||||
}
|
||||
|
||||
@@ -291,6 +291,7 @@ mod tests {
|
||||
window: (Timestamp(0), Timestamp(0)),
|
||||
seed: 0,
|
||||
broker: "t".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics: summarize(&[], &[]),
|
||||
}
|
||||
|
||||
@@ -115,6 +115,7 @@ fn run_point(point: &[Cell]) -> RunReport {
|
||||
window: (Timestamp(0), Timestamp(0)),
|
||||
seed: 0,
|
||||
broker: "test".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics: summarize(&equity, &exposure),
|
||||
}
|
||||
|
||||
@@ -66,6 +66,7 @@ fn run_point(server: &Arc<DataServer>, point: &[Cell], from: Timestamp, to: Time
|
||||
window: (from, to),
|
||||
seed: 0,
|
||||
broker: "sim-optimal(pip_size=1)".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics: summarize(&equity, &exposure),
|
||||
}
|
||||
|
||||
@@ -76,6 +76,7 @@ fn run_point(
|
||||
window: (from, to),
|
||||
seed: 0,
|
||||
broker: "sim-optimal(pip_size=1)".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics: summarize(&equity, &exposure),
|
||||
};
|
||||
|
||||
@@ -399,6 +399,7 @@ pub fn report_from_trace(trace: &[BarTrace], from: Timestamp, to: Timestamp) ->
|
||||
window: (from, to),
|
||||
seed: 0,
|
||||
broker: "sim-optimal(pip_size=1)".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics,
|
||||
}
|
||||
|
||||
@@ -68,6 +68,7 @@ fn run_point(server: &Arc<DataServer>, point: &[Cell], from: Timestamp, to: Time
|
||||
window: (from, to),
|
||||
seed: 0,
|
||||
broker: "sim-optimal(pip_size=1)".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics: summarize(&equity, &exposure),
|
||||
}
|
||||
|
||||
@@ -89,6 +89,7 @@ fn run_sample_over(prices: Vec<(Timestamp, Scalar)>) -> RunReport {
|
||||
window,
|
||||
seed: 0,
|
||||
broker: "sim-optimal(pip_size=0.0001)".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics,
|
||||
}
|
||||
|
||||
@@ -89,6 +89,7 @@ fn run_sample(window: (Timestamp, Timestamp), source: Box<dyn Source>) -> RunRep
|
||||
window,
|
||||
seed: 0,
|
||||
broker: "sim-optimal(pip_size=0.0001)".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics: summarize(&equity, &exposure),
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
//! (`Registry::append`, `RunReport::to_json`) is untouched and still emits the
|
||||
//! tagged form, so this is a one-directional widening, not a format change.
|
||||
|
||||
use aura_engine::{RunManifest, RunMetrics, RunReport, Scalar, Timestamp};
|
||||
use aura_engine::{FamilySelection, RunManifest, RunMetrics, RunReport, Scalar, Timestamp};
|
||||
use serde::de::{self, Deserializer};
|
||||
use serde::Deserialize;
|
||||
|
||||
@@ -30,6 +30,8 @@ struct RunManifestRead {
|
||||
window: (Timestamp, Timestamp),
|
||||
seed: u64,
|
||||
broker: String,
|
||||
#[serde(default)]
|
||||
selection: Option<FamilySelection>,
|
||||
}
|
||||
|
||||
/// A param value that accepts BOTH the current tagged [`Scalar`] form
|
||||
@@ -62,7 +64,7 @@ impl<'de> Deserialize<'de> for ScalarRead {
|
||||
|
||||
impl From<RunReportRead> for RunReport {
|
||||
fn from(r: RunReportRead) -> Self {
|
||||
let RunManifestRead { commit, params, window, seed, broker } = r.manifest;
|
||||
let RunManifestRead { commit, params, window, seed, broker, selection } = r.manifest;
|
||||
RunReport {
|
||||
manifest: RunManifest {
|
||||
commit,
|
||||
@@ -70,6 +72,7 @@ impl From<RunReportRead> for RunReport {
|
||||
window,
|
||||
seed,
|
||||
broker,
|
||||
selection,
|
||||
},
|
||||
metrics: r.metrics,
|
||||
}
|
||||
|
||||
+278
-42
@@ -17,7 +17,10 @@ use std::fs;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use aura_engine::{RunReport, SweepFamily, SweepPoint};
|
||||
use aura_engine::{
|
||||
expected_max_of_normals, r_metrics_from_rs, resample_block, FamilySelection, MetricStats,
|
||||
RunReport, SelectionMode, SplitMix64, SweepFamily, SweepPoint,
|
||||
};
|
||||
|
||||
mod compat;
|
||||
|
||||
@@ -107,25 +110,10 @@ enum Metric {
|
||||
NetExpectancyR,
|
||||
}
|
||||
|
||||
/// The per-metric **best-first ordering** of two reports: `Less` means `a` is
|
||||
/// better than `b`. "Best" is fixed by each metric's meaning: `total_pips` and
|
||||
/// the four R keys (`sqn`, `sqn_normalized`, `expectancy_r`, `net_expectancy_r`) higher-is-better;
|
||||
/// `max_drawdown` and `bias_sign_flips` lower-is-better. The top-level `f64` keys
|
||||
/// (`total_pips`, `max_drawdown`) use `partial_cmp` — finite by construction, so
|
||||
/// the `Ordering` always exists. The R keys live in an optional `metrics.r` block
|
||||
/// and use `total_cmp`: a member with `r: None` injects `f64::NEG_INFINITY` (the
|
||||
/// worst rank for higher-is-better), and `total_cmp` orders that — and any stray
|
||||
/// `NaN` — deterministically without panicking. An unknown metric name is a
|
||||
/// `RegistryError::UnknownMetric`.
|
||||
///
|
||||
/// The single source of truth for "best" — both [`rank_by`] (sort by it) and
|
||||
/// [`optimize`] (argmax by it) call this, so the per-metric direction lives in
|
||||
/// exactly one place. The metric name is matched once here; the returned closure
|
||||
/// carries the resolved `Metric`, so per-comparison work is just the key compare.
|
||||
fn metric_cmp(
|
||||
metric: &str,
|
||||
) -> Result<impl Fn(&RunReport, &RunReport) -> Ordering, RegistryError> {
|
||||
let metric = match metric {
|
||||
/// Resolve a metric NAME to the closed `Metric` kind (the single name→kind map).
|
||||
/// An unknown name is a `RegistryError::UnknownMetric`.
|
||||
fn resolve_metric(name: &str) -> Result<Metric, RegistryError> {
|
||||
Ok(match name {
|
||||
"total_pips" => Metric::TotalPips,
|
||||
"max_drawdown" => Metric::MaxDrawdown,
|
||||
// accept the new name AND the pre-rename one (CLI back-compat).
|
||||
@@ -135,31 +123,52 @@ fn metric_cmp(
|
||||
"expectancy_r" => Metric::ExpectancyR,
|
||||
"net_expectancy_r" => Metric::NetExpectancyR,
|
||||
other => return Err(RegistryError::UnknownMetric(other.to_string())),
|
||||
};
|
||||
// The R metrics live in an optional block; a missing `r` sorts to the bottom
|
||||
// for these higher-is-better keys (NEG_INFINITY < any finite R).
|
||||
})
|
||||
}
|
||||
|
||||
/// The metric's scalar value for one report. R metrics live in the optional `r`
|
||||
/// block; a missing block reads `NEG_INFINITY` (the worst rank for the
|
||||
/// higher-is-better R keys), preserving `metric_cmp`'s documented contract.
|
||||
fn metric_value(rep: &RunReport, m: Metric) -> f64 {
|
||||
fn r_get(rep: &RunReport, f: impl Fn(&aura_engine::RMetrics) -> f64) -> f64 {
|
||||
rep.metrics.r.as_ref().map(f).unwrap_or(f64::NEG_INFINITY)
|
||||
}
|
||||
Ok(move |a: &RunReport, b: &RunReport| match metric {
|
||||
// higher-is-better -> better when greater
|
||||
Metric::TotalPips => b.metrics.total_pips.partial_cmp(&a.metrics.total_pips).unwrap(),
|
||||
// lower-is-better -> better when smaller
|
||||
Metric::MaxDrawdown => a.metrics.max_drawdown.partial_cmp(&b.metrics.max_drawdown).unwrap(),
|
||||
Metric::BiasSignFlips => {
|
||||
a.metrics.bias_sign_flips.cmp(&b.metrics.bias_sign_flips)
|
||||
}
|
||||
// higher-is-better; total_cmp so NEG_INFINITY orders deterministically and
|
||||
// no NaN can panic.
|
||||
Metric::Sqn => r_get(b, |r| r.sqn).total_cmp(&r_get(a, |r| r.sqn)),
|
||||
Metric::SqnNormalized => {
|
||||
r_get(b, |r| r.sqn_normalized).total_cmp(&r_get(a, |r| r.sqn_normalized))
|
||||
}
|
||||
Metric::ExpectancyR => {
|
||||
r_get(b, |r| r.expectancy_r).total_cmp(&r_get(a, |r| r.expectancy_r))
|
||||
}
|
||||
Metric::NetExpectancyR => {
|
||||
r_get(b, |r| r.net_expectancy_r).total_cmp(&r_get(a, |r| r.net_expectancy_r))
|
||||
match m {
|
||||
Metric::TotalPips => rep.metrics.total_pips,
|
||||
Metric::MaxDrawdown => rep.metrics.max_drawdown,
|
||||
Metric::BiasSignFlips => rep.metrics.bias_sign_flips as f64,
|
||||
Metric::Sqn => r_get(rep, |r| r.sqn),
|
||||
Metric::SqnNormalized => r_get(rep, |r| r.sqn_normalized),
|
||||
Metric::ExpectancyR => r_get(rep, |r| r.expectancy_r),
|
||||
Metric::NetExpectancyR => r_get(rep, |r| r.net_expectancy_r),
|
||||
}
|
||||
}
|
||||
|
||||
/// The per-metric **best-first ordering** of two reports: `Less` means `a` is
|
||||
/// better than `b`. "Best" is fixed by each metric's meaning: `total_pips` and
|
||||
/// the four R keys (`sqn`, `sqn_normalized`, `expectancy_r`, `net_expectancy_r`)
|
||||
/// higher-is-better; `max_drawdown` and `bias_sign_flips` lower-is-better. Every
|
||||
/// metric compares via `total_cmp` (a total order over `f64`): top-level keys are
|
||||
/// finite by construction, while the R keys live in an optional `metrics.r` block
|
||||
/// where a member with `r: None` reads `f64::NEG_INFINITY` (the worst rank for
|
||||
/// higher-is-better) — `total_cmp` orders that, and any stray `NaN`,
|
||||
/// deterministically without panicking. An unknown metric name is a
|
||||
/// `RegistryError::UnknownMetric`.
|
||||
///
|
||||
/// The single source of truth for "best" — both [`rank_by`] (sort by it) and
|
||||
/// [`optimize`] (argmax by it) call this, so the per-metric direction lives in
|
||||
/// exactly one place. The metric name is resolved once via `resolve_metric`; the
|
||||
/// returned closure carries the resolved `Metric`, so per-comparison work is just
|
||||
/// reading each value (`metric_value`) and the key compare.
|
||||
fn metric_cmp(metric: &str) -> Result<impl Fn(&RunReport, &RunReport) -> Ordering, RegistryError> {
|
||||
let m = resolve_metric(metric)?;
|
||||
Ok(move |a: &RunReport, b: &RunReport| {
|
||||
let (va, vb) = (metric_value(a, m), metric_value(b, m));
|
||||
match m {
|
||||
// lower-is-better
|
||||
Metric::MaxDrawdown | Metric::BiasSignFlips => va.total_cmp(&vb),
|
||||
// higher-is-better
|
||||
_ => vb.total_cmp(&va),
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -193,6 +202,106 @@ pub fn optimize(family: &SweepFamily, metric: &str) -> Result<SweepPoint, Regist
|
||||
Ok(winner.clone())
|
||||
}
|
||||
|
||||
fn is_r_metric(m: Metric) -> bool {
|
||||
matches!(m, Metric::Sqn | Metric::SqnNormalized | Metric::ExpectancyR | Metric::NetExpectancyR)
|
||||
}
|
||||
|
||||
fn member_trade_rs(rep: &RunReport) -> &[f64] {
|
||||
rep.metrics.r.as_ref().map(|r| r.trade_rs.as_slice()).unwrap_or(&[])
|
||||
}
|
||||
|
||||
/// Recompute the selected R metric from a trade-R slice (reuses the engine's
|
||||
/// `r_metrics_from_rs`, the same arithmetic `summarize_r` uses).
|
||||
fn member_metric_from_rs(rs: &[f64], m: Metric) -> f64 {
|
||||
let rm = r_metrics_from_rs(rs);
|
||||
match m {
|
||||
Metric::Sqn => rm.sqn,
|
||||
Metric::SqnNormalized => rm.sqn_normalized,
|
||||
Metric::ExpectancyR => rm.expectancy_r,
|
||||
Metric::NetExpectancyR => rm.net_expectancy_r,
|
||||
_ => unreachable!("member_metric_from_rs is R-only"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The centred best-of-K null-max distribution: each member's `trade_rs` is
|
||||
/// mean-subtracted (the no-edge null), then for each resample every member is
|
||||
/// moving-block-resampled (in odometer order, from a per-iteration seed) and the
|
||||
/// max recomputed metric across members is taken. Deterministic given `seed` (C1).
|
||||
fn null_best_of_k(family: &SweepFamily, m: Metric, n_resamples: usize, block_len: usize, seed: u64) -> Vec<f64> {
|
||||
let centred: Vec<Vec<f64>> = family.points.iter().map(|p| {
|
||||
let rs = member_trade_rs(&p.report);
|
||||
if rs.is_empty() { return Vec::new(); }
|
||||
let mean = rs.iter().sum::<f64>() / rs.len() as f64;
|
||||
rs.iter().map(|x| x - mean).collect()
|
||||
}).collect();
|
||||
(0..n_resamples).map(|i| {
|
||||
let mut rng = SplitMix64::new(seed ^ i as u64);
|
||||
centred.iter().fold(f64::NEG_INFINITY, |best, rs| {
|
||||
if rs.is_empty() { return best; }
|
||||
let bl = block_len.clamp(1, rs.len());
|
||||
let v = member_metric_from_rs(&resample_block(rs, bl, &mut rng), m);
|
||||
best.max(v)
|
||||
})
|
||||
}).collect()
|
||||
}
|
||||
|
||||
/// Sample standard deviation of the K members' metric values (the `total_pips`
|
||||
/// dispersion-floor arm). `< 2` members -> `0.0`.
|
||||
fn member_sd(family: &SweepFamily, m: Metric) -> f64 {
|
||||
let vals: Vec<f64> = family.points.iter().map(|p| metric_value(&p.report, m)).collect();
|
||||
let n = vals.len();
|
||||
if n < 2 { return 0.0; }
|
||||
let mean = vals.iter().sum::<f64>() / n as f64;
|
||||
(vals.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / (n - 1) as f64).sqrt()
|
||||
}
|
||||
|
||||
/// `optimize`'s argmax winner PLUS its trials-deflation provenance. The returned
|
||||
/// `SweepPoint` is byte-identical to `optimize(family, metric)` (additive, C23).
|
||||
/// R arm: a centred moving-block reality-check (`overfit_probability` =
|
||||
/// `(count(null ≥ raw) + 1)/(n + 1)`, `deflated_score` = `raw − p95(null)`).
|
||||
/// `total_pips` arm: a closed-form expected-max-of-K dispersion floor, no
|
||||
/// probability. Deterministic given `seed` (C1).
|
||||
pub fn optimize_deflated(
|
||||
family: &SweepFamily, metric: &str, n_resamples: usize, block_len: usize, seed: u64,
|
||||
) -> Result<(SweepPoint, FamilySelection), RegistryError> {
|
||||
let winner = optimize(family, metric)?;
|
||||
let m = resolve_metric(metric)?;
|
||||
let k = family.points.len();
|
||||
let raw = metric_value(&winner.report, m);
|
||||
|
||||
let (deflated_score, overfit_probability) = if is_r_metric(m) {
|
||||
let null_max = null_best_of_k(family, m, n_resamples, block_len, seed);
|
||||
// Keep only the finite best-of-K draws. The null is *not computable* in two
|
||||
// degenerate cases: zero resamples (`null_max` empty), or no member carries
|
||||
// `trade_rs` (every member's centred series is empty, so each iteration's
|
||||
// best-of-K folds to `NEG_INFINITY`). The latter is reachable — `trade_rs`
|
||||
// is `#[serde(skip)]`, so a family loaded back from the registry has empty
|
||||
// conduits even when its `r` block (hence `raw`) is finite. Filtering to
|
||||
// finite values collapses both into one "no usable null" branch, instead of
|
||||
// letting `p95 = NEG_INFINITY` turn `deflated_score` into `+inf`.
|
||||
let usable: Vec<f64> = null_max.into_iter().filter(|x| x.is_finite()).collect();
|
||||
if usable.is_empty() {
|
||||
// No reality-check is computable: floor to a degenerate-but-defined
|
||||
// result — no deflation credit (`deflated_score == raw`) and a
|
||||
// maximally-uncertain overfit probability (1.0), never a nonsensical inf.
|
||||
(raw, Some(1.0))
|
||||
} else {
|
||||
let p95 = MetricStats::from_values(&usable).p95;
|
||||
let over = (usable.iter().filter(|&&x| x >= raw).count() + 1) as f64
|
||||
/ (usable.len() + 1) as f64;
|
||||
(raw - p95, Some(over))
|
||||
}
|
||||
} else {
|
||||
(raw - member_sd(family, m) * expected_max_of_normals(k), None)
|
||||
};
|
||||
|
||||
Ok((winner.clone(), FamilySelection {
|
||||
selection_metric: metric.to_string(), n_trials: k, raw_winner_metric: raw,
|
||||
deflated_score, overfit_probability, mode: SelectionMode::Argmax,
|
||||
n_resamples, block_len, seed,
|
||||
}))
|
||||
}
|
||||
|
||||
/// What can go wrong reading or ranking the registry.
|
||||
#[derive(Debug)]
|
||||
pub enum RegistryError {
|
||||
@@ -242,6 +351,7 @@ mod tests {
|
||||
window: (Timestamp(1), Timestamp(2)),
|
||||
seed: 0,
|
||||
broker: "b".to_string(),
|
||||
selection: None,
|
||||
},
|
||||
metrics: RunMetrics { total_pips, max_drawdown, bias_sign_flips: flips, r: None },
|
||||
}
|
||||
@@ -393,6 +503,19 @@ mod tests {
|
||||
let _ = fs::remove_file(&path);
|
||||
}
|
||||
|
||||
/// Property: the compat read-path carries a manifest's `selection` block
|
||||
/// through to the canonical `RunReport` — a line written *with* a
|
||||
/// FamilySelection lifts it intact (not dropped) through `RunReportRead`'s
|
||||
/// structural mirror and its `From` conversion.
|
||||
#[test]
|
||||
fn load_lifts_a_selection_block_through_the_compat_mirror() {
|
||||
let line = r#"{"manifest":{"commit":"c","params":[],"window":[0,0],"seed":0,"broker":"b","selection":{"selection_metric":"sqn_normalized","n_trials":4,"raw_winner_metric":1.8,"deflated_score":0.2,"overfit_probability":0.06,"mode":"Argmax","n_resamples":1000,"block_len":5,"seed":42}},"metrics":{"total_pips":0.0,"max_drawdown":0.0,"bias_sign_flips":0}}"#;
|
||||
let rep: RunReport = serde_json::from_str::<crate::compat::RunReportRead>(line).unwrap().into();
|
||||
let sel = rep.manifest.selection.expect("selection lifted");
|
||||
assert_eq!(sel.n_trials, 4);
|
||||
assert_eq!(sel.mode, aura_engine::SelectionMode::Argmax);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_by_orders_best_first_per_metric() {
|
||||
let reports = vec![
|
||||
@@ -547,4 +670,117 @@ mod tests {
|
||||
let ranked = rank_by(member_reports, "total_pips").expect("rank");
|
||||
assert_eq!(ranked[0].metrics.total_pips, 3.0); // best-first within the family
|
||||
}
|
||||
|
||||
fn member(total_pips: f64, trade_rs: Vec<f64>) -> SweepPoint {
|
||||
// Every RMetrics field is derived from the R slice by `r_metrics_from_rs`
|
||||
// (the same arithmetic production members carry), then the per-trade
|
||||
// `trade_rs` conduit is restored: `r_metrics_from_rs` empties it
|
||||
// (`trade_rs: Vec::new()`), whereas a production member is built by
|
||||
// `summarize_r`, which RETAINS it — and `optimize_deflated`'s R arm
|
||||
// resamples exactly that conduit, so a fixture without it cannot reach
|
||||
// the R-arm path under test.
|
||||
let mut r = aura_engine::r_metrics_from_rs(&trade_rs);
|
||||
r.trade_rs = trade_rs;
|
||||
SweepPoint {
|
||||
params: vec![],
|
||||
report: RunReport {
|
||||
manifest: RunManifest {
|
||||
commit: "c".into(), params: vec![], window: (Timestamp(0), Timestamp(0)),
|
||||
seed: 0, broker: "b".into(), selection: None,
|
||||
},
|
||||
metrics: RunMetrics {
|
||||
total_pips, max_drawdown: 0.0, bias_sign_flips: 0,
|
||||
r: Some(r),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture_family_with_r() -> SweepFamily {
|
||||
SweepFamily { space: vec![], points: vec![
|
||||
member(10.0, vec![1.0, -0.5, 0.8, -0.2, 0.6]),
|
||||
member(30.0, vec![2.0, 1.5, -0.3, 1.0, 0.9]), // best by every metric here
|
||||
member(5.0, vec![-0.5, 0.2, -0.8, 0.1, -0.3]),
|
||||
]}
|
||||
}
|
||||
|
||||
fn fixture_family_one_strong_edge() -> SweepFamily {
|
||||
SweepFamily { space: vec![], points: vec![
|
||||
member(1.0, vec![2.0, 2.5, 1.8, 2.2, 2.1, 1.9]), // strong, consistent +R edge
|
||||
member(1.0, vec![0.05, -0.1, 0.0, 0.1, -0.05, 0.02]), // ~zero-edge noise
|
||||
member(1.0, vec![-0.1, 0.1, 0.0, -0.05, 0.05, 0.0]),
|
||||
]}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optimize_deflated_winner_is_byte_identical_to_optimize() {
|
||||
let fam = fixture_family_with_r(); // helper: ≥3 members, varied sqn_normalized + trade_rs
|
||||
for metric in ["total_pips", "sqn_normalized", "expectancy_r"] {
|
||||
let plain = optimize(&fam, metric).unwrap();
|
||||
let (defl, _) = optimize_deflated(&fam, metric, 200, 3, 7).unwrap();
|
||||
assert_eq!(defl, plain, "deflation must not change the winner ({metric})");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optimize_deflated_is_reproducible_from_its_seed() {
|
||||
let fam = fixture_family_with_r();
|
||||
let a = optimize_deflated(&fam, "sqn_normalized", 500, 4, 42).unwrap().1;
|
||||
let b = optimize_deflated(&fam, "sqn_normalized", 500, 4, 42).unwrap().1;
|
||||
assert_eq!(a, b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optimize_deflated_total_pips_arm_floors_without_a_probability() {
|
||||
let fam = fixture_family_with_r();
|
||||
let sel = optimize_deflated(&fam, "total_pips", 100, 3, 1).unwrap().1;
|
||||
assert!(sel.overfit_probability.is_none());
|
||||
assert_eq!(sel.selection_metric, "total_pips");
|
||||
assert!(sel.deflated_score <= sel.raw_winner_metric); // dispersion floor ≤ raw
|
||||
}
|
||||
|
||||
/// Sibling-contract parity with `r_bootstrap`, which defines `n_resamples ==
|
||||
/// 0` as a valid input that floors to an all-zero result. `optimize_deflated`'s
|
||||
/// R arm must not panic on an empty null distribution: with zero resamples it
|
||||
/// returns a degenerate-but-defined provenance — `deflated_score == raw` (the
|
||||
/// p95 of an empty null is 0), `overfit_probability == 1` (the `(0 + 1)/(0 +
|
||||
/// 1)` Laplace floor) — never indexing the empty `MetricStats::from_values`.
|
||||
#[test]
|
||||
fn optimize_deflated_zero_resamples_floors_like_r_bootstrap() {
|
||||
let fam = fixture_family_with_r();
|
||||
let sel = optimize_deflated(&fam, "expectancy_r", 0, 3, 1).unwrap().1;
|
||||
assert_eq!(sel.n_resamples, 0);
|
||||
assert_eq!(sel.deflated_score, sel.raw_winner_metric, "p95 of an empty null is 0");
|
||||
assert_eq!(sel.overfit_probability, Some(1.0), "(0+1)/(0+1) Laplace floor");
|
||||
}
|
||||
|
||||
/// Regression: a family whose members carry an `r` block but NO `trade_rs` —
|
||||
/// the serde-skipped state of a family `load`ed back from the registry — must
|
||||
/// NOT yield a `+inf` deflated score on the R arm with positive resamples. With
|
||||
/// no member contributing a centred series, every best-of-K draw is
|
||||
/// `NEG_INFINITY`; the finite-filter collapses this to the same degenerate floor
|
||||
/// as zero-resamples (`deflated_score == raw`, `overfit_probability == 1.0`).
|
||||
#[test]
|
||||
fn optimize_deflated_no_trade_rs_floors_instead_of_infinity() {
|
||||
let mut fam = fixture_family_with_r();
|
||||
for p in &mut fam.points {
|
||||
if let Some(r) = p.report.metrics.r.as_mut() {
|
||||
r.trade_rs.clear(); // mimic a serde-loaded member (trade_rs is #[serde(skip)])
|
||||
}
|
||||
}
|
||||
let sel = optimize_deflated(&fam, "expectancy_r", 500, 3, 1).unwrap().1;
|
||||
assert!(sel.deflated_score.is_finite(), "deflated_score must be finite, got {}", sel.deflated_score);
|
||||
assert_eq!(sel.deflated_score, sel.raw_winner_metric);
|
||||
assert_eq!(sel.overfit_probability, Some(1.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optimize_deflated_overfit_probability_is_low_for_a_clear_edge() {
|
||||
// A family whose winner has a strong real R-edge over near-zero-edge peers:
|
||||
// the centred best-of-K null rarely reaches the winner's raw metric.
|
||||
let fam = fixture_family_one_strong_edge();
|
||||
let sel = optimize_deflated(&fam, "expectancy_r", 1000, 1, 9).unwrap().1;
|
||||
assert!(sel.overfit_probability.unwrap() < 0.2, "p={:?}", sel.overfit_probability);
|
||||
assert!(sel.deflated_score > 0.0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -253,6 +253,7 @@ mod tests {
|
||||
window: (Timestamp(1), Timestamp(5)),
|
||||
seed: 0,
|
||||
broker: "sim-optimal(pip_size=0.0001)".to_string(),
|
||||
selection: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user