diff --git a/crates/aura-analysis/src/lib.rs b/crates/aura-analysis/src/lib.rs index c99bed1..6c4590e 100644 --- a/crates/aura-analysis/src/lib.rs +++ b/crates/aura-analysis/src/lib.rs @@ -234,6 +234,45 @@ pub fn permute(xs: &mut [T], rng: &mut SplitMix64) { } } +/// Per-metric vocabulary a report payload supplies to the registry's +/// ranking/deflation machinery (#147). Narrow by design (#136): exactly the +/// surface the backtest metrics (the sole implementor so far) and the +/// measurement IC (deflatable today via its own permutation-null path, not +/// yet through this trait) are expected to share. The registry owns the +/// family fold — best-of-K, p95, Laplace, dispersion floor — the payload +/// owns what each metric IS: its name, direction, value read, and what one +/// draw of its null statistic means. +pub trait MetricVocabulary: Sized { + /// Resolved metric identity — cheap to copy, resolved once per call. + type Key: Copy; + /// Name → key; `None` for a name outside this vocabulary. + fn resolve(name: &str) -> Option; + /// The canonical accepted-name roster, in refusal-prose order. + fn known() -> &'static [&'static str]; + /// The key's optimisation direction: `true` when larger is better. + fn higher_is_better(key: Self::Key) -> bool; + /// The member's scalar under the key. A missing optional block reads + /// `f64::NEG_INFINITY` (the worst rank for higher-is-better keys). + fn value(&self, key: Self::Key) -> f64; + /// Whether the key deflates via a resampled null (the best-of-K arm) or + /// the dispersion floor. + fn has_resampling_null(key: Self::Key) -> bool; + /// ONE draw of this member's null statistic under H0, advancing `rng`. + /// `None` means no usable null input for this member, and consumes no + /// rng. `block_len` parameterises resampling vocabularies; permutation + /// vocabularies ignore it. + fn null_draw(&self, key: Self::Key, block_len: usize, rng: &mut SplitMix64) -> Option; +} + +/// One-sided Laplace-smoothed tail estimate `(ge + 1) / (n + 1)`: the +/// probability mass at-or-above an observed statistic given that `ge` of `n` +/// null draws reached it. Intended as the single source of the formula the +/// registry's R deflation arm and the CLI's IC reduction each compute inline +/// today; wiring those call sites to this function is deferred follow-up. +pub fn one_sided_p_laplace(ge: usize, n: usize) -> f64 { + (ge + 1) as f64 / (n + 1) as f64 +} + #[cfg(test)] mod tests { use super::*; @@ -356,4 +395,13 @@ mod tests { filled += take; } } + + #[test] + fn one_sided_p_laplace_known_values() { + assert_eq!(one_sided_p_laplace(0, 3), 0.25); + assert_eq!(one_sided_p_laplace(3, 3), 1.0); + assert_eq!(one_sided_p_laplace(1, 999), 2.0 / 1000.0); + // the registry's R-arm shape: count-of-ge over usable draws + assert_eq!(one_sided_p_laplace(0, 999), 1.0 / 1000.0); + } } diff --git a/crates/aura-backtest/src/lib.rs b/crates/aura-backtest/src/lib.rs index 529508e..5d7a66c 100644 --- a/crates/aura-backtest/src/lib.rs +++ b/crates/aura-backtest/src/lib.rs @@ -17,7 +17,7 @@ mod sim_broker; pub use mc::{monte_carlo, r_bootstrap, McAggregate, McDraw, McFamily, RBootstrap}; pub use metrics::{ derive_position_events, r_metrics_from_rs, summarize, summarize_r, PositionAction, - PositionEvent, RMetrics, RunMetrics, + PositionEvent, RMetrics, RunMetricKey, RunMetrics, RANKABLE_METRICS, R_BASED_METRICS, }; pub use position_management::{ ExitReason, FIELD_NAMES as PM_FIELD_NAMES, PositionManagement, diff --git a/crates/aura-backtest/src/metrics.rs b/crates/aura-backtest/src/metrics.rs index 3847399..3310900 100644 --- a/crates/aura-backtest/src/metrics.rs +++ b/crates/aura-backtest/src/metrics.rs @@ -9,6 +9,7 @@ //! re-associated. use aura_core::{Scalar, SeriesFold, Timestamp}; +use aura_engine::{resample_block, MetricVocabulary, SplitMix64}; /// Reduce a run's recorded pip-equity + exposure streams into summary metrics. /// Pure — identical inputs yield identical metrics (C1/C12). Timestamps are @@ -476,6 +477,125 @@ pub struct PositionEvent { pub volume: f64, } +/// The backtest metric vocabulary's resolved keys (was aura-registry's +/// private `Metric` enum — the vocabulary is supplied by backtest now, not +/// baked into the registry; #147). The runtime hot path branches on this +/// rather than re-parsing the name. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum RunMetricKey { + TotalPips, + MaxDrawdown, + BiasSignFlips, + Sqn, + SqnNormalized, + ExpectancyR, + NetExpectancyR, +} + +impl RunMetricKey { + /// C10: the four R-denominated keys — the account/instrument-agnostic + /// set the R-only registry gates (`check_r_metric`, `generalization`) + /// accept. + pub fn r_based(self) -> bool { + matches!( + self, + Self::Sqn | Self::SqnNormalized | Self::ExpectancyR | Self::NetExpectancyR + ) + } +} + +/// The four R-based names, in refusal-prose order (the NonRMetric message). +pub const R_BASED_METRICS: &[&str] = + &["sqn", "sqn_normalized", "expectancy_r", "net_expectancy_r"]; + +/// The rankable roster, in refusal-prose order — the roster this vocabulary +/// resolves against (#147). The registry's own refusal prose and +/// aura-campaign's re-export still hand-copy this list (not yet single- +/// sourced from here); wiring them to read from here is deferred follow-up. +pub const RANKABLE_METRICS: &[&str] = &[ + "total_pips", + "max_drawdown", + "bias_sign_flips", + "sqn", + "sqn_normalized", + "expectancy_r", + "net_expectancy_r", +]; + +/// Recompute the selected R metric from a trade-R slice (reuses +/// `r_metrics_from_rs`, the same arithmetic `summarize_r` uses) — the +/// deflation null's member reduction (was aura-registry-private, #147). +fn member_metric_from_rs(rs: &[f64], key: RunMetricKey) -> f64 { + let rm = r_metrics_from_rs(rs); + match key { + RunMetricKey::Sqn => rm.sqn, + RunMetricKey::SqnNormalized => rm.sqn_normalized, + RunMetricKey::ExpectancyR => rm.expectancy_r, + RunMetricKey::NetExpectancyR => rm.net_expectancy_r, + _ => unreachable!("member_metric_from_rs is R-only"), + } +} + +impl MetricVocabulary for RunMetrics { + type Key = RunMetricKey; + + fn resolve(name: &str) -> Option { + Some(match name { + "total_pips" => RunMetricKey::TotalPips, + "max_drawdown" => RunMetricKey::MaxDrawdown, + // accept the new name AND the pre-rename one (CLI back-compat). + "bias_sign_flips" | "exposure_sign_flips" => RunMetricKey::BiasSignFlips, + "sqn" => RunMetricKey::Sqn, + "sqn_normalized" => RunMetricKey::SqnNormalized, + "expectancy_r" => RunMetricKey::ExpectancyR, + "net_expectancy_r" => RunMetricKey::NetExpectancyR, + _ => return None, + }) + } + + fn known() -> &'static [&'static str] { + RANKABLE_METRICS + } + + fn higher_is_better(key: RunMetricKey) -> bool { + !matches!(key, RunMetricKey::MaxDrawdown | RunMetricKey::BiasSignFlips) + } + + fn value(&self, key: RunMetricKey) -> f64 { + fn r_get(m: &RunMetrics, f: impl Fn(&RMetrics) -> f64) -> f64 { + m.r.as_ref().map(f).unwrap_or(f64::NEG_INFINITY) + } + match key { + RunMetricKey::TotalPips => self.total_pips, + RunMetricKey::MaxDrawdown => self.max_drawdown, + RunMetricKey::BiasSignFlips => self.bias_sign_flips as f64, + RunMetricKey::Sqn => r_get(self, |r| r.sqn), + RunMetricKey::SqnNormalized => r_get(self, |r| r.sqn_normalized), + RunMetricKey::ExpectancyR => r_get(self, |r| r.expectancy_r), + RunMetricKey::NetExpectancyR => r_get(self, |r| r.net_expectancy_r), + } + } + + fn has_resampling_null(key: RunMetricKey) -> bool { + key.r_based() + } + + /// # Panics + /// Panics (via `member_metric_from_rs`'s `unreachable!`) if `key` is not + /// R-based — callers must gate on `has_resampling_null(key)` before + /// drawing, exactly as documented at the trait level. + fn null_draw(&self, key: RunMetricKey, block_len: usize, rng: &mut SplitMix64) -> Option { + let rs: &[f64] = self.r.as_ref().map(|r| r.net_trade_rs.as_slice()).unwrap_or(&[]); + if rs.is_empty() { + return None; // consumes no rng — bit-parity with the pre-#147 fold + } + let mean = rs.iter().sum::() / rs.len() as f64; + let centred: Vec = rs.iter().map(|x| x - mean).collect(); + let bl = block_len.clamp(1, rs.len()); + Some(member_metric_from_rs(&resample_block(¢red, bl, rng), key)) + } +} + #[cfg(test)] mod tests { use super::*; @@ -1252,4 +1372,102 @@ mod tests { let m = summarize(&[], &exposure); assert_eq!(m.bias_sign_flips, 0); } + + #[test] + fn run_metrics_vocabulary_resolves_and_classifies() { + use aura_engine::MetricVocabulary; + assert!(matches!( + ::resolve("sqn"), + Some(RunMetricKey::Sqn) + )); + // the pre-rename CLI alias must keep resolving (back-compat). + assert!(matches!( + ::resolve("exposure_sign_flips"), + Some(RunMetricKey::BiasSignFlips) + )); + assert!(::resolve("sharpe").is_none()); + assert_eq!(::known(), RANKABLE_METRICS); + assert!(RunMetricKey::Sqn.r_based()); + assert!(!RunMetricKey::TotalPips.r_based()); + } + + /// Property: an R-based key on a member with no `r` block ranks worst for + /// a higher-is-better metric (`f64::NEG_INFINITY`), never a silent default + /// like `0.0` that could out-rank a genuinely poor member; non-R keys are + /// read straight from the pip-only fields regardless of the `r` block. + #[test] + fn value_r_based_key_reads_neg_infinity_when_r_block_absent() { + let m = RunMetrics { total_pips: 5.0, max_drawdown: 2.0, bias_sign_flips: 1, r: None }; + for key in [ + RunMetricKey::Sqn, + RunMetricKey::SqnNormalized, + RunMetricKey::ExpectancyR, + RunMetricKey::NetExpectancyR, + ] { + assert_eq!(m.value(key), f64::NEG_INFINITY, "missing r block must rank worst"); + } + assert_eq!(m.value(RunMetricKey::TotalPips), 5.0); + assert_eq!(m.value(RunMetricKey::MaxDrawdown), 2.0); + assert_eq!(m.value(RunMetricKey::BiasSignFlips), 1.0); + } + + /// Property: a member with no trade-R input (no `r` block) yields no null + /// draw AND leaves the rng untouched — the "consumes no rng" contract the + /// implementation comment claims. Verified indirectly (SplitMix64 carries + /// no Clone/PartialEq): the rng's next draw after the no-op call matches a + /// freshly, identically-seeded rng's first draw. + #[test] + fn null_draw_returns_none_and_consumes_no_rng_when_no_trades() { + let m = RunMetrics { total_pips: 0.0, max_drawdown: 0.0, bias_sign_flips: 0, r: None }; + let mut rng = SplitMix64::new(11); + let got = m.null_draw(RunMetricKey::Sqn, 2, &mut rng); + assert_eq!(got, None); + let mut fresh = SplitMix64::new(11); + assert_eq!( + rng.next_u64(), + fresh.next_u64(), + "null_draw's None path must consume no rng state" + ); + } + + /// Property: `null_draw` recomputes the selected key by mean-centring + /// `net_trade_rs`, block-resampling it under the given rng, and reducing + /// via `member_metric_from_rs` — exactly the recipe reproduced here by + /// hand with an identically-seeded rng (C1: same seed -> same draw). + #[test] + fn null_draw_recomputes_the_selected_key_from_a_seeded_block_resample() { + let net_trade_rs = vec![1.0, 2.0, -1.0, 3.0]; + let m = RunMetrics { + total_pips: 0.0, + max_drawdown: 0.0, + bias_sign_flips: 0, + r: Some(RMetrics { + expectancy_r: 0.0, + n_trades: 4, + win_rate: 0.0, + avg_win_r: 0.0, + avg_loss_r: 0.0, + profit_factor: 0.0, + max_r_drawdown: 0.0, + n_open_at_end: 0, + sqn: 0.0, + sqn_normalized: 0.0, + net_expectancy_r: 0.0, + conviction_terciles_r: [0.0; 3], + net_trade_rs: net_trade_rs.clone(), + }), + }; + let mut rng_a = SplitMix64::new(7); + let got = m + .null_draw(RunMetricKey::ExpectancyR, 2, &mut rng_a) + .expect("nonempty net_trade_rs yields a draw"); + + let mean = net_trade_rs.iter().sum::() / net_trade_rs.len() as f64; + let centred: Vec = net_trade_rs.iter().map(|x| x - mean).collect(); + let mut rng_b = SplitMix64::new(7); + let resampled = resample_block(¢red, 2, &mut rng_b); + let want = member_metric_from_rs(&resampled, RunMetricKey::ExpectancyR); + + assert_eq!(got, want); + } } diff --git a/crates/aura-engine/src/lib.rs b/crates/aura-engine/src/lib.rs index 8d7af9f..6828679 100644 --- a/crates/aura-engine/src/lib.rs +++ b/crates/aura-engine/src/lib.rs @@ -79,7 +79,7 @@ pub use sweep::{ // `resample_block`/`MetricStats` are foundation-grade (aura-analysis, #291); // re-exported here (unchanged exported name set) so existing // `aura_engine::{resample_block, MetricStats}` consumers stay source-compatible. -pub use aura_analysis::{resample_block, MetricStats}; +pub use aura_analysis::{one_sided_p_laplace, resample_block, MetricStats, MetricVocabulary}; pub use walkforward::{ param_stability, walk_forward, RollMode, WalkForwardError, WalkForwardResult, WindowBounds, WindowOutcome, WindowRoller, WindowRun, diff --git a/crates/aura-registry/src/lib.rs b/crates/aura-registry/src/lib.rs index dde0dd2..2c64f0b 100644 --- a/crates/aura-registry/src/lib.rs +++ b/crates/aura-registry/src/lib.rs @@ -18,8 +18,8 @@ use std::io::Write; use std::path::{Path, PathBuf}; use aura_core::PrimitiveBuilder; -use aura_analysis::{resample_block, MetricStats, SplitMix64}; -use aura_backtest::{r_metrics_from_rs, RunReport, SweepFamily, SweepPoint}; +use aura_analysis::{one_sided_p_laplace, MetricStats, MetricVocabulary, SplitMix64}; +use aura_backtest::{RunMetrics, RunReport}; use aura_engine::{ blueprint_from_json, blueprint_identity_json, expected_max_of_normals, FamilySelection, SelectionMode, @@ -495,77 +495,30 @@ impl Registry { } } -/// Which metric a best-first comparison keys on. Resolves a metric *name* (a -/// string at the API boundary) to a closed kind once, so the per-comparison hot -/// path branches on this rather than re-parsing the name. -#[derive(Clone, Copy)] -enum Metric { - TotalPips, - MaxDrawdown, - BiasSignFlips, - Sqn, - SqnNormalized, - ExpectancyR, - NetExpectancyR, -} - -/// Resolve a metric NAME to the closed `Metric` kind (the single name→kind map). -/// An unknown name is a `RegistryError::UnknownMetric`. -fn resolve_metric(name: &str) -> Result { - Ok(match name { - "total_pips" => Metric::TotalPips, - "max_drawdown" => Metric::MaxDrawdown, - // accept the new name AND the pre-rename one (CLI back-compat). - "bias_sign_flips" | "exposure_sign_flips" => Metric::BiasSignFlips, - "sqn" => Metric::Sqn, - "sqn_normalized" => Metric::SqnNormalized, - "expectancy_r" => Metric::ExpectancyR, - "net_expectancy_r" => Metric::NetExpectancyR, - other => return Err(RegistryError::UnknownMetric(other.to_string())), - }) -} - -/// The metric's scalar value for one report. R metrics live in the optional `r` -/// block; a missing block reads `NEG_INFINITY` (the worst rank for the -/// higher-is-better R keys), preserving `metric_cmp`'s documented contract. -fn metric_value(rep: &RunReport, m: Metric) -> f64 { - fn r_get(rep: &RunReport, f: impl Fn(&aura_backtest::RMetrics) -> f64) -> f64 { - rep.metrics.r.as_ref().map(f).unwrap_or(f64::NEG_INFINITY) - } - match m { - Metric::TotalPips => rep.metrics.total_pips, - Metric::MaxDrawdown => rep.metrics.max_drawdown, - Metric::BiasSignFlips => rep.metrics.bias_sign_flips as f64, - Metric::Sqn => r_get(rep, |r| r.sqn), - Metric::SqnNormalized => r_get(rep, |r| r.sqn_normalized), - Metric::ExpectancyR => r_get(rep, |r| r.expectancy_r), - Metric::NetExpectancyR => r_get(rep, |r| r.net_expectancy_r), - } -} - /// The per-metric **best-first ordering** of two reports: `Less` means `a` is -/// better than `b`. "Best" is fixed by each metric's meaning: `total_pips` and -/// the four R keys (`sqn`, `sqn_normalized`, `expectancy_r`, `net_expectancy_r`) -/// higher-is-better; `max_drawdown` and `bias_sign_flips` lower-is-better. Every -/// metric compares via `total_cmp` (a total order over `f64`): top-level keys are -/// finite by construction, while the R keys live in an optional `metrics.r` block -/// where a member with `r: None` reads `f64::NEG_INFINITY` (the worst rank for -/// higher-is-better) — `total_cmp` orders that, and any stray `NaN`, -/// deterministically without panicking. An unknown metric name is a -/// `RegistryError::UnknownMetric`. +/// better than `b`. "Best" is fixed by each metric's meaning as declared by +/// the payload's `MetricVocabulary` (direction via `higher_is_better`; a +/// missing optional block reads `f64::NEG_INFINITY`, the worst rank for +/// higher-is-better keys). Every metric compares via `total_cmp` (a total +/// order over `f64`), so strays order deterministically without panicking. An +/// unknown metric name is a `RegistryError::UnknownMetric` carrying the +/// vocabulary's own roster. /// -/// The single source of best-first *ordering* — both [`rank_by`] (sort by it) and -/// [`optimize`] (argmax by it) call this. The per-metric direction itself lives in -/// `higher_is_better` (which this and `optimize_plateau` both read); `metric_cmp` -/// turns that direction into a `total_cmp` over the read values. The metric name is -/// resolved once via `resolve_metric`; the returned closure carries the resolved -/// `Metric`, so per-comparison work is just reading each value (`metric_value`) and -/// the key compare. -fn metric_cmp(metric: &str) -> Result Ordering, RegistryError> { - let m = resolve_metric(metric)?; - let hib = higher_is_better(m); - Ok(move |a: &RunReport, b: &RunReport| { - let (va, vb) = (metric_value(a, m), metric_value(b, m)); +/// The single source of best-first *ordering* — both [`rank_by`] (sort by it) +/// and [`optimize`] (argmax by it) call this. The name is resolved once via +/// `M::resolve`; the returned closure carries the resolved key, so +/// per-comparison work is just reading each value and the key compare. +fn metric_cmp( + metric: &str, +) -> Result, &aura_engine::RunReport) -> Ordering, RegistryError> +{ + let key = M::resolve(metric).ok_or_else(|| RegistryError::UnknownMetric { + name: metric.to_string(), + known: M::known(), + })?; + let hib = M::higher_is_better(key); + Ok(move |a: &aura_engine::RunReport, b: &aura_engine::RunReport| { + let (va, vb) = (a.metrics.value(key), b.metrics.value(key)); // higher-is-better ranks the larger value first; lower-is-better the smaller. if hib { vb.total_cmp(&va) } else { va.total_cmp(&vb) } }) @@ -575,8 +528,11 @@ fn metric_cmp(metric: &str) -> Result Orderin /// by each metric's meaning (see `metric_cmp`). A stable sort keeps file /// (insertion) order among ties. An unknown metric name is a /// `RegistryError::UnknownMetric`. -pub fn rank_by(mut reports: Vec, metric: &str) -> Result, RegistryError> { - let cmp = metric_cmp(metric)?; +pub fn rank_by( + mut reports: Vec>, + metric: &str, +) -> Result>, RegistryError> { + let cmp = metric_cmp::(metric)?; reports.sort_by(|a, b| cmp(a, b)); Ok(reports) } @@ -590,8 +546,11 @@ pub fn rank_by(mut reports: Vec, metric: &str) -> Result Result { - let cmp = metric_cmp(metric)?; +pub fn optimize( + family: &aura_engine::SweepFamily, + metric: &str, +) -> Result, RegistryError> { + let cmp = metric_cmp::(metric)?; let winner = family .points .iter() @@ -600,19 +559,6 @@ pub fn optimize(family: &SweepFamily, metric: &str) -> Result bool { - matches!(m, Metric::Sqn | Metric::SqnNormalized | Metric::ExpectancyR | Metric::NetExpectancyR) -} - -/// The metric's optimisation direction: `true` if a larger value is better -/// (`total_pips`, the four R keys), `false` for the lower-is-better keys -/// (`max_drawdown`, `bias_sign_flips`). The single direction source — `metric_cmp` -/// (best-first ordering) and `optimize_plateau` (smoothed argmax + worst-neighbour) -/// both read it, so the per-metric sense lives in exactly one place. -fn higher_is_better(m: Metric) -> bool { - !matches!(m, Metric::MaxDrawdown | Metric::BiasSignFlips) -} - /// The closed grid neighbourhood of flat index `i` over a mixed-radix lattice /// (`axis_lens`, last axis fastest — the `GridSpace` odometer convention): `{i}` /// plus each in-range ±1-per-axis cell. Pure index math, deterministic. The order @@ -641,49 +587,36 @@ fn closed_neighbourhood(i: usize, axis_lens: &[usize]) -> Vec { out } -fn member_net_trade_rs(rep: &RunReport) -> &[f64] { - rep.metrics.r.as_ref().map(|r| r.net_trade_rs.as_slice()).unwrap_or(&[]) -} - -/// Recompute the selected R metric from a trade-R slice (reuses the engine's -/// `r_metrics_from_rs`, the same arithmetic `summarize_r` uses). -fn member_metric_from_rs(rs: &[f64], m: Metric) -> f64 { - let rm = r_metrics_from_rs(rs); - match m { - Metric::Sqn => rm.sqn, - Metric::SqnNormalized => rm.sqn_normalized, - Metric::ExpectancyR => rm.expectancy_r, - Metric::NetExpectancyR => rm.net_expectancy_r, - _ => unreachable!("member_metric_from_rs is R-only"), - } -} - -/// The centred best-of-K null-max distribution: each member's `net_trade_rs` is -/// mean-subtracted (the no-edge null), then for each resample every member is -/// moving-block-resampled (in odometer order, from a per-iteration seed) and the -/// max recomputed metric across members is taken. Deterministic given `seed` (C1). -fn null_best_of_k(family: &SweepFamily, m: Metric, n_resamples: usize, block_len: usize, seed: u64) -> Vec { - let centred: Vec> = family.points.iter().map(|p| { - let rs = member_net_trade_rs(&p.report); - if rs.is_empty() { return Vec::new(); } - let mean = rs.iter().sum::() / rs.len() as f64; - rs.iter().map(|x| x - mean).collect() - }).collect(); - (0..n_resamples).map(|i| { - let mut rng = SplitMix64::new(seed ^ i as u64); - centred.iter().fold(f64::NEG_INFINITY, |best, rs| { - if rs.is_empty() { return best; } - let bl = block_len.clamp(1, rs.len()); - let v = member_metric_from_rs(&resample_block(rs, bl, &mut rng), m); - best.max(v) +/// The best-of-K null-max distribution, vocabulary-generic: for each resample +/// every member draws ONE null statistic from its own vocabulary (in odometer +/// order, from a per-iteration seed) and the max across members is taken. A +/// member with no usable null input contributes nothing and consumes no rng +/// (bit-parity with the pre-#147 R fold, whose centring is now inside +/// `RunMetrics::null_draw`). Deterministic given `seed` (C1). +fn null_best_of_k( + family: &aura_engine::SweepFamily, + key: M::Key, + n_resamples: usize, + block_len: usize, + seed: u64, +) -> Vec { + (0..n_resamples) + .map(|i| { + let mut rng = SplitMix64::new(seed ^ i as u64); + family.points.iter().fold(f64::NEG_INFINITY, |best, p| { + match p.report.metrics.null_draw(key, block_len, &mut rng) { + Some(v) => best.max(v), + None => best, + } + }) }) - }).collect() + .collect() } /// Sample standard deviation of the K members' metric values (the `total_pips` /// dispersion-floor arm). `< 2` members -> `0.0`. -fn member_sd(family: &SweepFamily, m: Metric) -> f64 { - let vals: Vec = family.points.iter().map(|p| metric_value(&p.report, m)).collect(); +fn member_sd(family: &aura_engine::SweepFamily, key: M::Key) -> f64 { + let vals: Vec = family.points.iter().map(|p| p.report.metrics.value(key)).collect(); let n = vals.len(); if n < 2 { return 0.0; } let mean = vals.iter().sum::() / n as f64; @@ -702,27 +635,30 @@ pub enum PlateauMode { /// `optimize`'s argmax over the NEIGHBOURHOOD-SMOOTHED surface, plus its plateau /// provenance. Each member scores as the mean (or worst-case) of its closed grid -/// neighbourhood's `metric_value`; the winner is the best smoothed score by the +/// neighbourhood's metric value; the winner is the best smoothed score by the /// metric's own direction (earliest-odometer tie, as `optimize`). `axis_lens` are /// the grid radixes (`param_space()` order, last axis fastest). Pure, /// deterministic (C1). The returned `SweepPoint` is the winning grid CELL; /// `raw_winner_metric` is that cell's own metric, `neighbourhood_score` the /// smoothed value the argmax maximised. -pub fn optimize_plateau( - family: &SweepFamily, axis_lens: &[usize], metric: &str, mode: PlateauMode, -) -> Result<(SweepPoint, FamilySelection), RegistryError> { - let m = resolve_metric(metric)?; +pub fn optimize_plateau( + family: &aura_engine::SweepFamily, axis_lens: &[usize], metric: &str, mode: PlateauMode, +) -> Result<(aura_engine::SweepPoint, FamilySelection), RegistryError> { + let key = M::resolve(metric).ok_or_else(|| RegistryError::UnknownMetric { + name: metric.to_string(), + known: M::known(), + })?; let n = family.points.len(); debug_assert_eq!( axis_lens.iter().product::(), n, "axis_lens product must equal the family size (a valid grid lattice)", ); - let hib = higher_is_better(m); + let hib = M::higher_is_better(key); // smoothed score + closed-neighbourhood size per member, in odometer order let scored: Vec<(usize, f64, usize)> = (0..n) .map(|i| { let nbrs = closed_neighbourhood(i, axis_lens); - let vals: Vec = nbrs.iter().map(|&j| metric_value(&family.points[j].report, m)).collect(); + let vals: Vec = nbrs.iter().map(|&j| family.points[j].report.metrics.value(key)).collect(); let score = match mode { PlateauMode::Mean => vals.iter().sum::() / vals.len() as f64, PlateauMode::Worst => { @@ -746,7 +682,7 @@ pub fn optimize_plateau( }) .expect("a SweepFamily is non-empty by construction (EmptyAxis is rejected upstream)"); let winner = family.points[wi].clone(); - let raw = metric_value(&winner.report, m); + let raw = winner.report.metrics.value(key); Ok(( winner, FamilySelection { @@ -777,28 +713,34 @@ pub const DEFLATION_BLOCK_LEN: usize = 5; /// `optimize`'s argmax winner PLUS its trials-deflation provenance. The returned /// `SweepPoint` is byte-identical to `optimize(family, metric)` (additive, C23). -/// R arm: a centred moving-block reality-check (`overfit_probability` = -/// `(count(null ≥ raw) + 1)/(n + 1)`, `deflated_score` = `raw − p95(null)`). -/// `total_pips` arm: a closed-form expected-max-of-K dispersion floor, no -/// probability. Deterministic given `seed` (C1). -pub fn optimize_deflated( - family: &SweepFamily, metric: &str, n_resamples: usize, block_len: usize, seed: u64, -) -> Result<(SweepPoint, FamilySelection), RegistryError> { +/// Which arm runs is vocabulary-declared resampling null vs dispersion floor +/// (`M::has_resampling_null`): a resampling-null key gets a centred moving-block +/// reality-check (`overfit_probability` = `(count(null ≥ raw) + 1)/(n + 1)`, +/// `deflated_score` = `raw − p95(null)`); otherwise a closed-form +/// expected-max-of-K dispersion floor, no probability. Deterministic given +/// `seed` (C1). +pub fn optimize_deflated( + family: &aura_engine::SweepFamily, metric: &str, n_resamples: usize, block_len: usize, seed: u64, +) -> Result<(aura_engine::SweepPoint, FamilySelection), RegistryError> { let winner = optimize(family, metric)?; - let m = resolve_metric(metric)?; + let key = M::resolve(metric).ok_or_else(|| RegistryError::UnknownMetric { + name: metric.to_string(), + known: M::known(), + })?; let k = family.points.len(); - let raw = metric_value(&winner.report, m); + let raw = winner.report.metrics.value(key); - let (deflated_score, overfit_probability) = if is_r_metric(m) { - let null_max = null_best_of_k(family, m, n_resamples, block_len, seed); + let (deflated_score, overfit_probability) = if M::has_resampling_null(key) { + let null_max = null_best_of_k(family, key, n_resamples, block_len, seed); // Keep only the finite best-of-K draws. The null is *not computable* in two - // degenerate cases: zero resamples (`null_max` empty), or no member carries - // `net_trade_rs` (every member's centred series is empty, so each iteration's - // best-of-K folds to `NEG_INFINITY`). The latter is reachable — `net_trade_rs` - // is `#[serde(skip)]`, so a family loaded back from the registry has empty - // conduits even when its `r` block (hence `raw`) is finite. Filtering to - // finite values collapses both into one "no usable null" branch, instead of - // letting `p95 = NEG_INFINITY` turn `deflated_score` into `+inf`. + // degenerate cases: zero resamples (`null_max` empty), or no member yields a + // usable draw (`null_draw` returns `None` for every member, so each + // iteration's best-of-K folds to `NEG_INFINITY`). The latter is reachable — + // the R conduit `net_trade_rs` is `#[serde(skip)]`, so a family loaded back + // from the registry has empty conduits even when its `r` block (hence `raw`) + // is finite. Filtering to finite values collapses both into one "no usable + // null" branch, instead of letting `p95 = NEG_INFINITY` turn + // `deflated_score` into `+inf`. let usable: Vec = null_max.into_iter().filter(|x| x.is_finite()).collect(); if usable.is_empty() { // No reality-check is computable: floor to a degenerate-but-defined @@ -807,22 +749,22 @@ pub fn optimize_deflated( (raw, Some(1.0)) } else { let p95 = MetricStats::from_values(&usable).p95; - let over = (usable.iter().filter(|&&x| x >= raw).count() + 1) as f64 - / (usable.len() + 1) as f64; + let ge = usable.iter().filter(|&&x| x >= raw).count(); + let over = one_sided_p_laplace(ge, usable.len()); (raw - p95, Some(over)) } } else { // The dispersion floor SUBTRACTS the expected-max inflation, so it is valid // only for a higher-is-better metric (a larger value is the better one the // search inflated). A lower-is-better metric would deflate wrong-signed; the - // two shipped call sites only pass `total_pips`, so this is unreached, but + // shipped call sites only pass `total_pips`, so this is unreached, but // `optimize_deflated` is public — guard the assumption rather than leave it // implicit. debug_assert!( - !matches!(m, Metric::MaxDrawdown | Metric::BiasSignFlips), + M::higher_is_better(key), "dispersion-floor arm is for higher-is-better metrics only", ); - (raw - member_sd(family, m) * expected_max_of_normals(k), None) + (raw - member_sd(family, key) * expected_max_of_normals(k), None) }; Ok((winner.clone(), FamilySelection { @@ -854,8 +796,13 @@ pub struct Generalization { /// the CLI runs before evaluating any instrument. The registry owns metric truth /// (C9), so the CLI never duplicates the R-set. pub fn check_r_metric(metric: &str) -> Result<(), RegistryError> { - let m = resolve_metric(metric)?; - if is_r_metric(m) { + let key = ::resolve(metric).ok_or_else(|| { + RegistryError::UnknownMetric { + name: metric.to_string(), + known: ::known(), + } + })?; + if key.r_based() { Ok(()) } else { Err(RegistryError::NonRMetric(metric.to_string())) @@ -871,8 +818,13 @@ pub fn generalization( per_instrument: &[(String, &RunReport)], metric: &str, ) -> Result { - let m = resolve_metric(metric)?; - if !is_r_metric(m) { + let key = ::resolve(metric).ok_or_else(|| { + RegistryError::UnknownMetric { + name: metric.to_string(), + known: ::known(), + } + })?; + if !key.r_based() { return Err(RegistryError::NonRMetric(metric.to_string())); } if per_instrument.len() < 2 { @@ -880,7 +832,7 @@ pub fn generalization( } let vals: Vec<(String, f64)> = per_instrument .iter() - .map(|(label, rep)| (label.clone(), metric_value(rep, m))) + .map(|(label, rep)| (label.clone(), rep.metrics.value(key))) .collect(); let worst_case = vals.iter().map(|(_, v)| *v).fold(f64::INFINITY, f64::min); let sign_agreement = vals.iter().filter(|(_, v)| *v > 0.0).count(); @@ -900,8 +852,8 @@ pub enum RegistryError { Io(std::io::Error), /// A stored line did not parse as a `RunReport` (1-based line number). Parse { line: usize, source: serde_json::Error }, - /// `rank_by` was given a metric name it does not know. - UnknownMetric(String), + /// `rank_by` was given a metric name outside the family's vocabulary. + UnknownMetric { name: String, known: &'static [&'static str] }, /// A known metric that is not R-based — cross-instrument generalization is /// R-only (R is the only account/instrument-agnostic unit, C10). NonRMetric(String), @@ -916,13 +868,15 @@ impl fmt::Display for RegistryError { RegistryError::Parse { line, source } => { write!(f, "registry parse error at line {line}: {source}") } - RegistryError::UnknownMetric(m) => write!( + RegistryError::UnknownMetric { name, known } => write!( f, - "unknown metric '{m}' (known: total_pips, max_drawdown, bias_sign_flips, sqn, sqn_normalized, expectancy_r, net_expectancy_r)" + "unknown metric '{name}' (known: {})", + known.join(", ") ), RegistryError::NonRMetric(m) => write!( f, - "metric '{m}' is not comparable across instruments; cross-instrument scoring is R-only (sqn, sqn_normalized, expectancy_r, net_expectancy_r)" + "metric '{m}' is not comparable across instruments; cross-instrument scoring is R-only ({})", + aura_backtest::R_BASED_METRICS.join(", ") ), RegistryError::TooFewInstruments(n) => write!( f, @@ -944,7 +898,7 @@ impl From for RegistryError { mod tests { use super::*; use aura_core::{Cell, Scalar, Timestamp}; - use aura_backtest::{RMetrics, RunMetrics}; + use aura_backtest::{RMetrics, RunMetrics, SweepFamily, SweepPoint}; use aura_engine::RunManifest; fn report_with(total_pips: f64, max_drawdown: f64, flips: u64) -> RunReport { @@ -1037,7 +991,8 @@ mod tests { #[test] fn unknown_metric_message_lists_r_metrics() { - let msg = RegistryError::UnknownMetric("nope".to_string()).to_string(); + let msg = RegistryError::UnknownMetric { name: "nope".to_string(), known: aura_backtest::RANKABLE_METRICS } + .to_string(); assert!(msg.contains("sqn"), "msg: {msg}"); assert!(msg.contains("sqn_normalized"), "msg: {msg}"); assert!(msg.contains("expectancy_r"), "msg: {msg}"); @@ -1176,7 +1131,7 @@ mod tests { fn rank_by_unknown_metric_is_an_error() { let reports = vec![report_with(1.0, 0.5, 1)]; match rank_by(reports, "sharpe") { - Err(RegistryError::UnknownMetric(m)) => assert_eq!(m, "sharpe"), + Err(RegistryError::UnknownMetric { name, .. }) => assert_eq!(name, "sharpe"), other => panic!("expected UnknownMetric, got {other:?}"), } } @@ -1589,8 +1544,8 @@ mod tests { let fam = fixture_family_with_r(); let (winner, sel) = optimize_deflated(&fam, "sqn_normalized", 200, 3, 1).unwrap(); assert_eq!(sel.n_trials, fam.points.len()); - let m = resolve_metric("sqn_normalized").unwrap(); - assert_eq!(sel.raw_winner_metric, metric_value(&winner.report, m)); + let m = ::resolve("sqn_normalized").unwrap(); + assert_eq!(sel.raw_winner_metric, winner.report.metrics.value(m)); } /// A 1-D grid (7 cells, odometer order) with two end spikes and a broad middle @@ -2373,7 +2328,7 @@ mod tests { other => panic!("expected NonRMetric, got {other:?}"), } match check_r_metric("nope") { - Err(RegistryError::UnknownMetric(m)) => assert_eq!(m, "nope"), + Err(RegistryError::UnknownMetric { name, .. }) => assert_eq!(name, "nope"), other => panic!("expected UnknownMetric, got {other:?}"), } }