From 0651e16146759f4c9d1c8c98e5fa09afc82837c0 Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 20 Jul 2026 19:55:24 +0200 Subject: [PATCH] feat: the IC becomes the second metric vocabulary; rosters single-sourced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The A1 cut of #147 item 2, part 2 of 2 (the consumers), completing what part 1 (the MetricVocabulary substrate) set up: - aura-campaign's RANKABLE_METRICS hand-copy becomes a re-export of the backtest vocabulary's roster; the #190 guard test extends to pin the roster nesting (rankable == vocabulary roster, rankable within per-member, per-member within the research vocabulary) and the legacy exposure_sign_flips alias, so any drift fails a test loudly. The research and per-member rosters themselves stay hand-copies by ratified isolation (#190 accepted residual) — now oracle-pinned instead of trusted. - aura-cli gains IcMetrics + IcKey implementing MetricVocabulary: the second PRODUCTION implementor. The standalone `aura measure ic` reduction now computes its permutation p via the shared null_draw + one_sided_p_laplace building blocks (independent draws instead of chained shuffles — statistically equivalent, deterministic per seed; emitted p values may differ in the last digits from #290's). - Acceptance proven at the registry boundary: an engineered SweepFamily deflates through optimize_deflated to a LOW overfit probability and a noise family to a HIGH one (the permutation null, dispatched by the vocabulary — the deflation machinery is no longer baked-in R-only); same seed reproduces the identical FamilySelection; cross-vocabulary names are refused with the family's own roster in the prose (the C10 guard). A new cli_run E2E exercises the legacy alias through the real `aura runs family … rank` boundary. Deviation from the plan, forced by the compiler: IcMetrics additionally derives Debug (unwrap_err on the deflation Result needs it; mirrors RunMetrics). Verification: full workspace suite green (no FAILED, no compile errors), clippy -D warnings clean, C18 goldens (cli_run, measure_ic, c28_layering) green unchanged. A2 — measurement runs as sweep-family citizens — stays deliberately deferred until a concrete campaign demand exists. closes #147 --- crates/aura-backtest/src/metrics.rs | 6 +- crates/aura-campaign/src/lib.rs | 12 +- .../tests/metric_vocabulary_e2e.rs | 28 ++++ crates/aura-cli/src/main.rs | 156 +++++++++++++++++- crates/aura-cli/tests/cli_run.rs | 64 +++++++ 5 files changed, 247 insertions(+), 19 deletions(-) diff --git a/crates/aura-backtest/src/metrics.rs b/crates/aura-backtest/src/metrics.rs index 3310900..d9414f7 100644 --- a/crates/aura-backtest/src/metrics.rs +++ b/crates/aura-backtest/src/metrics.rs @@ -509,9 +509,9 @@ pub const R_BASED_METRICS: &[&str] = &["sqn", "sqn_normalized", "expectancy_r", "net_expectancy_r"]; /// The rankable roster, in refusal-prose order — the roster this vocabulary -/// resolves against (#147). The registry's own refusal prose and -/// aura-campaign's re-export still hand-copy this list (not yet single- -/// sourced from here); wiring them to read from here is deferred follow-up. +/// resolves against (#147). The registry's refusal prose and +/// `aura_campaign::RANKABLE_METRICS` (a re-export) both derive from this +/// single source; neither hand-copies it anymore. pub const RANKABLE_METRICS: &[&str] = &[ "total_pips", "max_drawdown", diff --git a/crates/aura-campaign/src/lib.rs b/crates/aura-campaign/src/lib.rs index 5935226..b14f511 100644 --- a/crates/aura-campaign/src/lib.rs +++ b/crates/aura-campaign/src/lib.rs @@ -144,14 +144,10 @@ pub const PER_MEMBER_METRICS: &[&str] = &[ "sqn", "net_expectancy_r", ]; -/// The registry's rankable roster (`resolve_metric`'s name set) — the metrics -/// a sweep/walk_forward stage may select on. Hand-copied (#190); drift fails -/// safe (a name the registry would refuse is refused here first, before any -/// member runs). -pub const RANKABLE_METRICS: &[&str] = &[ - "total_pips", "max_drawdown", "bias_sign_flips", - "sqn", "sqn_normalized", "expectancy_r", "net_expectancy_r", -]; +/// The rankable subset a stage's `rank_by`/`select` may name — re-exported +/// from the backtest vocabulary (single source, #147): this roster and the +/// registry's refusal prose can no longer drift. +pub use aura_backtest::RANKABLE_METRICS; /// Deflation resample count — re-exported from `aura-registry`'s shared /// definition (single-sourced, #199). diff --git a/crates/aura-campaign/tests/metric_vocabulary_e2e.rs b/crates/aura-campaign/tests/metric_vocabulary_e2e.rs index c21f29c..e599858 100644 --- a/crates/aura-campaign/tests/metric_vocabulary_e2e.rs +++ b/crates/aura-campaign/tests/metric_vocabulary_e2e.rs @@ -107,3 +107,31 @@ fn metric_vocabulary_covers_all_shipped_scalar_metrics() { top-level enumeration)" ); } + +/// #147: the rankable roster is single-sourced from the vocabulary impl, the +/// rosters nest (rankable ⊆ per-member ⊆ research vocabulary), and the legacy +/// CLI alias keeps resolving. Any drift fails here, loudly. +#[test] +fn rankable_roster_is_single_sourced_and_nested() { + use aura_engine::MetricVocabulary; + + assert_eq!( + aura_campaign::RANKABLE_METRICS, + ::known(), + "campaign roster must BE the vocabulary's roster" + ); + for name in aura_campaign::RANKABLE_METRICS { + assert!( + aura_campaign::PER_MEMBER_METRICS.contains(name), + "rankable metric '{name}' missing from PER_MEMBER_METRICS" + ); + } + for name in aura_campaign::PER_MEMBER_METRICS { + assert!( + aura_research::metric_vocabulary().contains(name), + "per-member metric '{name}' missing from the research vocabulary" + ); + } + // the pre-rename CLI alias must keep resolving (back-compat). + assert!(::resolve("exposure_sign_flips").is_some()); +} diff --git a/crates/aura-cli/src/main.rs b/crates/aura-cli/src/main.rs index 484cf59..077f77e 100644 --- a/crates/aura-cli/src/main.rs +++ b/crates/aura-cli/src/main.rs @@ -35,7 +35,7 @@ use aura_backtest::{ RunMetrics, RunReport, SimBroker, SweepFamily, SweepPoint, WalkForwardResult, WindowRun, PM_FIELD_NAMES, PM_RECORD_KINDS, }; -use aura_analysis::{pearson_corr, permute, SplitMix64}; +use aura_analysis::{one_sided_p_laplace, pearson_corr, permute, MetricVocabulary, SplitMix64}; use aura_std::{ GatedRecorder, LinComb, Recorder, RollingMax, RollingMin, SeriesReducer, Sub, }; @@ -1844,6 +1844,51 @@ fn measurement_manifest( } } +/// The IC measurement payload: the scalar plus its in-memory null inputs +/// (the aligned pairs ride #[serde(skip)], mirroring RMetrics.net_trade_rs). +/// The second production implementor of `MetricVocabulary` (#147) — the +/// registry's deflation machinery dispatches to ITS permutation null. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +struct IcMetrics { + information_coefficient: f64, + #[serde(skip)] + sigs: Vec, + #[serde(skip)] + frs: Vec, +} + +/// The IC vocabulary's single resolved key. +#[derive(Clone, Copy)] +struct IcKey; + +impl MetricVocabulary for IcMetrics { + type Key = IcKey; + + fn resolve(name: &str) -> Option { + (name == "information_coefficient").then_some(IcKey) + } + fn known() -> &'static [&'static str] { + &["information_coefficient"] + } + fn higher_is_better(_: IcKey) -> bool { + true + } + fn value(&self, _: IcKey) -> f64 { + self.information_coefficient + } + fn has_resampling_null(_: IcKey) -> bool { + true + } + fn null_draw(&self, _: IcKey, _block_len: usize, rng: &mut SplitMix64) -> Option { + if self.sigs.len() < 2 { + return None; // consumes no rng + } + let mut perm = self.sigs.clone(); // independent draw, not a chained shuffle + permute(&mut perm, rng); + Some(pearson_corr(&perm, &self.frs)) + } +} + /// The pure IC reduction result (no CLI context) — unit-testable in isolation. /// Exercised by the in-crate `ic_tests` module and consumed by the run/command-path /// wiring in `dispatch_measure_ic`, which constructs `IcReport` from a live @@ -1879,8 +1924,9 @@ impl IcReport { /// price ts-spine (a post-run array shift over recorded data — C2 governs in-graph /// nodes, not a completed run's trace); signal is aligned to it by EXACT timestamp /// match (unmatched dropped). `< 2` aligned pairs or zero variance → the degenerate -/// floor `(0.0, 1.0)`. One-sided null (higher-is-better), Laplace-smoothed like the -/// R deflation arm. +/// floor `(0.0, 1.0)`. One-sided permutation null via the shared +/// `MetricVocabulary::null_draw` + `one_sided_p_laplace` building blocks (#147); +/// draws are independent permutations, deterministic given `seed`. fn information_coefficient( signal: &[(Timestamp, f64)], price: &[(Timestamp, f64)], @@ -1912,17 +1958,18 @@ fn information_coefficient( return IcOutcome { information_coefficient: 0.0, overfit_probability: 1.0, n_pairs }; } let raw = pearson_corr(&sigs, &frs); - // permutation null: shuffle signal against fixed returns, one-sided count + let member = IcMetrics { information_coefficient: raw, sigs, frs }; let mut rng = SplitMix64::new(seed); - let mut perm = sigs.clone(); let mut ge = 0usize; for _ in 0..permutations { - permute(&mut perm, &mut rng); - if pearson_corr(&perm, &frs) >= raw { + let draw = member + .null_draw(IcKey, 0, &mut rng) + .expect("n_pairs >= 2 checked above"); + if draw >= raw { ge += 1; } } - let overfit_probability = (ge + 1) as f64 / (permutations as f64 + 1.0); + let overfit_probability = one_sided_p_laplace(ge, permutations); IcOutcome { information_coefficient: raw, overfit_probability, n_pairs } } @@ -6790,4 +6837,97 @@ mod ic_tests { assert_eq!(out.n_pairs, 4, "only the 4 overlapping ts pair; price ts 0/3/6 and the decoys drop"); assert!(out.information_coefficient > 0.99, "ic = {}", out.information_coefficient); } + + fn ic_report(sigs: Vec, frs: Vec) -> aura_engine::RunReport { + aura_engine::RunReport { + manifest: aura_engine::RunManifest { + commit: "c".to_string(), + params: vec![("p".to_string(), aura_core::Scalar::f64(1.0))], + defaults: vec![], + window: (aura_core::Timestamp(1), aura_core::Timestamp(2)), + seed: 0, + broker: "b".to_string(), + selection: None, + instrument: None, + topology_hash: None, + project: None, + }, + metrics: IcMetrics { + information_coefficient: aura_analysis::pearson_corr(&sigs, &frs), + sigs, + frs, + }, + } + } + + fn ic_family(members: Vec>) -> aura_engine::SweepFamily { + aura_engine::SweepFamily { + space: vec![], + points: members + .into_iter() + .enumerate() + .map(|(i, report)| aura_engine::SweepPoint { + params: vec![aura_core::Cell::from_f64(i as f64)], + report, + }) + .collect(), + } + } + + /// #147 acceptance: the registry's deflation dispatches to the IC + /// vocabulary's OWN permutation null — a genuinely predictive family + /// deflates to a low overfit probability, a noise family to a high one. + #[test] + fn ic_family_deflation_dispatches_the_permutation_null() { + let ramp: Vec = (0..24).map(|i| i as f64).collect(); + let engineered = ic_family(vec![ + ic_report(ramp.clone(), ramp.clone()), // corr = 1.0: a real edge + ic_report(ramp.clone(), ramp.iter().rev().cloned().collect()), // corr = -1.0 + ]); + let (winner, sel) = + aura_registry::optimize_deflated(&engineered, "information_coefficient", 500, 5, 0) + .expect("ic family deflates"); + assert!((winner.report.metrics.information_coefficient - 1.0).abs() < 1e-12); + let p = sel.overfit_probability.expect("resampling-null arm sets a probability"); + assert!(p < 0.05, "perfectly predictive family must not look like overfit: p={p}"); + + // orthogonal signal — IC exactly 0.0; permuted draws beat it ~half the time. + let noise = ic_family(vec![ic_report( + vec![0.03, 0.01, 0.04, 0.02, 0.05, 0.00, 0.06, -0.01], + vec![0.01, 0.02, 0.03, 0.04, -0.04, -0.03, -0.02, -0.01], + )]); + let (_, sel) = + aura_registry::optimize_deflated(&noise, "information_coefficient", 500, 5, 0) + .expect("noise family deflates"); + let p = sel.overfit_probability.expect("probability present"); + assert!(p > 0.1, "a noise family must carry a high overfit probability: p={p}"); + } + + /// Same seed → identical selection provenance (C1 through the generic path). + #[test] + fn ic_family_deflation_is_deterministic_given_seed() { + let ramp: Vec = (0..16).map(|i| i as f64).collect(); + let fam = ic_family(vec![ic_report(ramp.clone(), ramp)]); + let a = aura_registry::optimize_deflated(&fam, "information_coefficient", 300, 5, 7) + .expect("deflate a"); + let b = aura_registry::optimize_deflated(&fam, "information_coefficient", 300, 5, 7) + .expect("deflate b"); + assert_eq!(a.1, b.1, "same seed must yield identical FamilySelection"); + } + + /// C10 guard: cross-vocabulary names are refused, and the refusal names + /// the family's REAL vocabulary. + #[test] + fn cross_vocabulary_names_are_refused_both_ways() { + let ramp: Vec = (0..8).map(|i| i as f64).collect(); + let fam = ic_family(vec![ic_report(ramp.clone(), ramp)]); + let err = aura_registry::optimize_deflated(&fam, "sqn", 100, 5, 0).unwrap_err(); + let prose = err.to_string(); + assert!( + prose.contains("unknown metric 'sqn'") && prose.contains("information_coefficient"), + "refusal must name the IC vocabulary: {prose}" + ); + // and the R side still refuses the IC name with ITS roster: + assert!(aura_registry::check_r_metric("information_coefficient").is_err()); + } } diff --git a/crates/aura-cli/tests/cli_run.rs b/crates/aura-cli/tests/cli_run.rs index b50b099..b63c550 100644 --- a/crates/aura-cli/tests/cli_run.rs +++ b/crates/aura-cli/tests/cli_run.rs @@ -1146,6 +1146,70 @@ fn runs_families_list_and_per_family_rank_across_invocations() { let _ = std::fs::remove_dir_all(&cwd); } +/// #147: the pre-rename metric name `exposure_sign_flips` still resolves +/// through the real, user-facing `aura runs family rank ` path — +/// not just through a direct `MetricVocabulary::resolve` call in a unit test. +/// Task 4 re-exports `aura_campaign::RANKABLE_METRICS` from the single-sourced +/// backtest vocabulary instead of hand-copying it; were the vocabulary's +/// `resolve()` match arm for the legacy alias ever dropped in that refactor, +/// this is the only test that would catch it at the CLI's actual observable +/// boundary (exit code + stdout), rather than inside the crate that owns the +/// vocabulary. +#[test] +fn runs_family_rank_accepts_legacy_exposure_sign_flips_alias() { + let cwd = temp_cwd("runs-rank-legacy-alias"); + let fixture = format!("{}/examples/r_sma.json", env!("CARGO_MANIFEST_DIR")); + let sweep = Command::new(BIN) + .args([ + "sweep", &fixture, + "--axis", "sma_signal.fast.length=2,4", + "--axis", "sma_signal.slow.length=8,16", + ]) + .current_dir(&cwd) + .output() + .expect("spawn sweep"); + assert!(sweep.status.success(), "sweep stderr: {}", String::from_utf8_lossy(&sweep.stderr)); + + let rank = Command::new(BIN) + .args(["runs", "family", "sweep-0", "rank", "exposure_sign_flips"]) + .current_dir(&cwd) + .output() + .expect("spawn rank"); + assert!( + rank.status.success(), + "the legacy alias must still resolve, not usage-error: stderr={}", + String::from_utf8_lossy(&rank.stderr) + ); + let rank_out = String::from_utf8(rank.stdout).expect("utf-8"); + assert_eq!(rank_out.lines().count(), 4, "all 4 members rank: {rank_out:?}"); + // the renamed field is what actually ships in the JSON — the alias is a + // resolve()-only back door, never a second output key. + assert!( + rank_out.lines().all(|l| l.contains("\"bias_sign_flips\":")), + "output still serializes under the new key: {rank_out:?}" + ); + assert!( + !rank_out.contains("\"exposure_sign_flips\":"), + "the legacy key must never appear in output: {rank_out:?}" + ); + + let flips: Vec = rank_out + .lines() + .map(|l| { + let key = "\"bias_sign_flips\":"; + let start = l.find(key).expect("line has bias_sign_flips") + key.len(); + let tail = &l[start..]; + let end = tail.find([',', '}']).expect("token end"); + tail[..end].parse().expect("bias_sign_flips is an integer") + }) + .collect(); + // bias_sign_flips is a lower-is-better metric (fewer direction reversals), + // so `rank` orders ascending here — unlike total_pips's descending order. + assert!(flips.windows(2).all(|w| w[0] <= w[1]), "rank not ascending by the aliased metric: {flips:?}"); + + let _ = std::fs::remove_dir_all(&cwd); +} + #[test] fn runs_list_and_rank_are_retired_and_exit_two() { // `aura runs list` / `runs rank ` were retired (#73): families