diff --git a/crates/aura-campaign/tests/metric_vocabulary_e2e.rs b/crates/aura-campaign/tests/metric_vocabulary_e2e.rs new file mode 100644 index 0000000..5adcaba --- /dev/null +++ b/crates/aura-campaign/tests/metric_vocabulary_e2e.rs @@ -0,0 +1,107 @@ +//! Cross-crate guard for #190: pins `aura_research::metric_vocabulary()` (the +//! hand-written 17-name declared metric roster) against the ACTUAL scalar +//! fields the three metric-bearing `aura-analysis` types serialize — not a +//! second hand-list that could independently drift. `aura-campaign` is the +//! one crate that already depends on both `aura-analysis` (the types) and +//! `aura-research` (the vocabulary), so it is the only place this pin can be +//! written today (`aura-research` deliberately has no `aura-analysis` dep — +//! `metric_vocabulary`'s own doc comment names this as an accepted residual). + +use std::collections::BTreeSet; + +use aura_analysis::{summarize_r, FamilySelection, RunMetrics, SelectionMode}; + +/// The names of every field of a serialized sample whose JSON value is a +/// `Number` — i.e. a scalar metric readout, not a string/enum tag, a nested +/// object, or an array (per-tercile / per-trade series). +fn scalar_field_names(sample: &serde_json::Value) -> BTreeSet { + sample + .as_object() + .expect("a metric-bearing type serializes to a JSON object") + .iter() + .filter(|(_, v)| v.is_number()) + .map(|(k, _)| k.clone()) + .collect() +} + +/// Property: the declared metric vocabulary (`aura_research::metric_vocabulary()`) +/// is exhaustive against — and carries no stale entry beyond — the scalar +/// fields the three metric-bearing `aura-analysis` types +/// (`RMetrics`/`RunMetrics`/`FamilySelection`) actually ship, enumerated via +/// their real `serde::Serialize` impls rather than a second hand-copy. +/// +/// If a field is added, renamed, or removed on any of the three types without +/// updating `metric_vocabulary()`, this test goes red: a new metric becomes +/// un-gateable (present in the type, absent from the vocabulary), or a stale +/// entry survives the vocabulary after its field is gone (present in the +/// vocabulary, absent from every type). Today the two rosters agree, so this +/// is a drift guard, not a repro: it must pass on arrival. +#[test] +fn metric_vocabulary_covers_all_shipped_scalar_metrics() { + // RMetrics: the real empty-input reduction (`summarize_r(&[], &[])`) is a + // genuine producer, not a hand-typed literal — every scalar field is + // unconditionally present (no `Option`/`skip_serializing_if` on this type). + let r_metrics = summarize_r(&[], &[]); + // RunMetrics: `r: None` keeps the nested `RMetrics` block out of this + // sample's JSON entirely (an `Object`, not a `Number`, so it would be + // filtered out anyway; `None` just avoids double-counting its fields here). + let run_metrics = RunMetrics { total_pips: 0.0, max_drawdown: 0.0, bias_sign_flips: 0, r: None }; + // FamilySelection: an explicit literal that populates exactly the three + // annotation SCORES the vocabulary declares (`deflated_score`, + // `overfit_probability`, `neighbourhood_score`) and leaves their sibling + // annotation IDENTIFIERS (`n_resamples`/`block_len`/`seed`/`n_neighbours`) + // absent — real producers never mix an Argmax score with a Plateau score, + // but this is a serde-shape probe, not a reproduction of a real selection + // run, so the mix is only used to exercise every annotation field name once. + let selection = FamilySelection { + selection_metric: "expectancy_r".to_string(), + n_trials: 4, + raw_winner_metric: 1.0, + mode: SelectionMode::Argmax, + deflated_score: Some(0.5), + overfit_probability: Some(0.1), + n_resamples: None, + block_len: None, + seed: None, + neighbourhood_score: Some(0.9), + n_neighbours: None, + }; + + let mut scalar_fields = BTreeSet::new(); + scalar_fields.extend(scalar_field_names(&serde_json::to_value(&r_metrics).unwrap())); + scalar_fields.extend(scalar_field_names(&serde_json::to_value(&run_metrics).unwrap())); + + let mut selection_scalars = scalar_field_names(&serde_json::to_value(&selection).unwrap()); + // `n_trials` / `raw_winner_metric` are selection PROVENANCE (how many + // trials ran; the raw pre-deflation score) about which existing metric + // the selection optimized, not a metric readout in their own right -- + // structurally the same exclusion the `Number` filter above already + // applies to `selection_metric` (a `String` naming that same metric) and + // `mode` (the selection rule tag). Never part of the declared vocabulary. + selection_scalars.remove("n_trials"); + selection_scalars.remove("raw_winner_metric"); + scalar_fields.extend(selection_scalars); + + let vocabulary: BTreeSet = + aura_research::metric_vocabulary().iter().map(|s| s.to_string()).collect(); + + let missing_from_vocabulary: Vec<_> = scalar_fields.difference(&vocabulary).collect(); + let stale_in_vocabulary: Vec<_> = vocabulary.difference(&scalar_fields).collect(); + assert!( + missing_from_vocabulary.is_empty(), + "shipped scalar metric(s) {missing_from_vocabulary:?} are un-gateable: \ + absent from aura_research::metric_vocabulary()" + ); + assert!( + stale_in_vocabulary.is_empty(), + "aura_research::metric_vocabulary() entry/entries {stale_in_vocabulary:?} are stale: \ + no shipped scalar field of RMetrics/RunMetrics/FamilySelection carries that name" + ); + assert_eq!( + vocabulary.len(), + 17, + "sanity count: 11 RMetrics + 3 RunMetrics + 3 FamilySelection scalars \ + (a mismatched count here means a field is nested/missed by this flat, \ + top-level enumeration)" + ); +}