feat: measurement's first deflatable metric — the Information Coefficient
Give a measurement run a standalone post-run quality score: the Information
Coefficient (IC), corr(signal_t, forward_return_{t+h}), with a permutation
null model. Before this a measurement run persisted only tap names and series
(MeasurementReport) — inspectable but not rankable or deflatable. IC is the
first metric measurement supplies: the tap-to-scalar reduction the C28
process-column / metric-interface seam anticipates ("a named metric vocabulary
supplied by measurement and backtest instead of baked in R-only").
Surface: `aura measure ic <run> --signal <tap> --price <tap> [--horizon]
[--permutations] [--seed]`. It reads the run's two recorded tap traces, builds
the forward-return series offline over the recorded price spine, aligns
signal -> return by exact timestamp, computes the IC, and prints it with a
one-sided permutation-null overfit probability (Laplace-smoothed, mirroring the
R deflation arm's formula in optimize_deflated).
Placement follows the C28 ladder: the generic pieces — pearson_corr and a
Fisher-Yates permute over SplitMix64 — land in aura-analysis (the domain-free
statistics foundation, beside resample_block); the IC semantics (forward-return
horizon, the offline join) and the CLI verb land in aura-cli (the shell). The
run path (run_measurement), MeasurementReport, the trace store, and the registry
metric vocabulary are untouched — existing `aura run` output stays byte-identical
(the C18 golden run_prints_json_and_exits_zero stays green).
Causality (C2): the forward-return read close_{t+h} is over a completed run's
recorded trace, not an in-graph node. A look-ahead signal is structurally
impossible in a causal run, so the engineered-vs-noise acceptance property is
unit-tested over hand-built offline series (perfect signal -> IC ~= 1, p < 0.05;
a varying but exactly-uncorrelated signal -> IC = 0, not significant;
constant/degenerate -> the (0.0, 1.0) floor), while the E2E validates the CLI
wiring, report well-formedness, determinism, and the error paths over a real run.
This lands the second, structurally distinct null-model computation (permutation,
beside the moving-block bootstrap behind R). That makes the deferred registry
deflation-vocabulary abstraction (#147 item 2) demand-driven rather than
speculative — the next cycle.
Verified: cargo test --workspace green (incl. the new aura-analysis unit tests,
the aura-cli ic_* unit tests, and the measure_ic E2E); cargo clippy --workspace
--all-targets -D warnings clean; the C18 byte-identity golden unchanged.
closes #290
refs #147
This commit is contained in:
@@ -35,6 +35,7 @@ use aura_backtest::{
|
||||
RunMetrics, RunReport, SimBroker, SweepFamily, SweepPoint, WalkForwardResult, WindowRun,
|
||||
PM_FIELD_NAMES, PM_RECORD_KINDS,
|
||||
};
|
||||
use aura_analysis::{pearson_corr, permute, SplitMix64};
|
||||
use aura_std::{
|
||||
GatedRecorder, LinComb, Recorder, RollingMax, RollingMin, SeriesReducer, Sub,
|
||||
};
|
||||
@@ -1843,6 +1844,88 @@ fn measurement_manifest(
|
||||
}
|
||||
}
|
||||
|
||||
/// The pure IC reduction result (no CLI context) — unit-testable in isolation.
|
||||
/// Exercised by the in-crate `ic_tests` module and consumed by the run/command-path
|
||||
/// wiring in `dispatch_measure_ic`, which constructs `IcReport` from a live
|
||||
/// measurement run's recorded taps.
|
||||
struct IcOutcome {
|
||||
information_coefficient: f64,
|
||||
overfit_probability: f64,
|
||||
n_pairs: usize,
|
||||
}
|
||||
|
||||
/// One measurement run's IC scalar + its permutation-null significance, for stdout.
|
||||
#[derive(serde::Serialize)]
|
||||
struct IcReport {
|
||||
run: String,
|
||||
signal_tap: String,
|
||||
price_tap: String,
|
||||
horizon: usize,
|
||||
permutations: usize,
|
||||
seed: u64,
|
||||
n_pairs: usize,
|
||||
information_coefficient: f64,
|
||||
overfit_probability: f64,
|
||||
}
|
||||
|
||||
impl IcReport {
|
||||
fn to_json(&self) -> String {
|
||||
serde_json::to_string(self).expect("a finite IcReport always serializes")
|
||||
}
|
||||
}
|
||||
|
||||
/// `corr(signal_t, forward_return_{t+h})` over two recorded tap series, with a
|
||||
/// permutation null. `forward_return[i] = (price[i+h] - price[i]) / price[i]` on the
|
||||
/// price ts-spine (a post-run array shift over recorded data — C2 governs in-graph
|
||||
/// nodes, not a completed run's trace); signal is aligned to it by EXACT timestamp
|
||||
/// match (unmatched dropped). `< 2` aligned pairs or zero variance → the degenerate
|
||||
/// floor `(0.0, 1.0)`. One-sided null (higher-is-better), Laplace-smoothed like the
|
||||
/// R deflation arm.
|
||||
fn information_coefficient(
|
||||
signal: &[(Timestamp, f64)],
|
||||
price: &[(Timestamp, f64)],
|
||||
horizon: usize,
|
||||
permutations: usize,
|
||||
seed: u64,
|
||||
) -> IcOutcome {
|
||||
use std::collections::HashMap;
|
||||
// signal value by timestamp-i64 (`Timestamp` is a tuple struct over epoch-ns; a tap
|
||||
// fires at most once per ts). `t.0` is the epoch-ns i64.
|
||||
let sig_at: HashMap<i64, f64> = signal.iter().map(|(t, v)| (t.0, *v)).collect();
|
||||
// forward returns on the price spine, paired with the signal at the SAME ts
|
||||
let (mut sigs, mut frs) = (Vec::new(), Vec::new());
|
||||
if horizon >= 1 && price.len() > horizon {
|
||||
for i in 0..price.len() - horizon {
|
||||
let (t, p0) = (price[i].0, price[i].1);
|
||||
if p0 == 0.0 {
|
||||
continue; // undefined return
|
||||
}
|
||||
let fr = (price[i + horizon].1 - p0) / p0;
|
||||
if let Some(&s) = sig_at.get(&t.0) {
|
||||
sigs.push(s);
|
||||
frs.push(fr);
|
||||
}
|
||||
}
|
||||
}
|
||||
let n_pairs = sigs.len();
|
||||
if n_pairs < 2 {
|
||||
return IcOutcome { information_coefficient: 0.0, overfit_probability: 1.0, n_pairs };
|
||||
}
|
||||
let raw = pearson_corr(&sigs, &frs);
|
||||
// permutation null: shuffle signal against fixed returns, one-sided count
|
||||
let mut rng = SplitMix64::new(seed);
|
||||
let mut perm = sigs.clone();
|
||||
let mut ge = 0usize;
|
||||
for _ in 0..permutations {
|
||||
permute(&mut perm, &mut rng);
|
||||
if pearson_corr(&perm, &frs) >= raw {
|
||||
ge += 1;
|
||||
}
|
||||
}
|
||||
let overfit_probability = (ge + 1) as f64 / (permutations as f64 + 1.0);
|
||||
IcOutcome { information_coefficient: raw, overfit_probability, n_pairs }
|
||||
}
|
||||
|
||||
/// The bare measurement run (C28 phase 3): `run_signal_r` MINUS `wrap_r` and the
|
||||
/// eq/ex/r R-evaluation, KEEPING the declared-tap bind → drain → persist (C27).
|
||||
/// No broker, no risk executor, no per-cycle equity/exposure/r recorders — this
|
||||
@@ -2968,6 +3051,8 @@ enum Command {
|
||||
Campaign(research_docs::CampaignCmd),
|
||||
/// Inspect the project's data archive.
|
||||
Data(DataCmd),
|
||||
/// Reduce a measurement run's recorded taps to a signal-quality metric.
|
||||
Measure(MeasureCmd),
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
@@ -3032,6 +3117,34 @@ struct DataCoverageCmd {
|
||||
symbol: String,
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
struct MeasureCmd {
|
||||
#[command(subcommand)]
|
||||
command: MeasureCommand,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum MeasureCommand {
|
||||
/// Information Coefficient of a signal tap against forward returns of a price tap.
|
||||
Ic(MeasureIcCmd),
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
struct MeasureIcCmd {
|
||||
/// The persisted run name (its trace-store subdirectory).
|
||||
run: String,
|
||||
#[arg(long)]
|
||||
signal: String,
|
||||
#[arg(long)]
|
||||
price: String,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
horizon: usize,
|
||||
#[arg(long, default_value_t = 1000)]
|
||||
permutations: usize,
|
||||
#[arg(long, default_value_t = 0)]
|
||||
seed: u64,
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
#[command(args_conflicts_with_subcommands = true)]
|
||||
struct GraphCmd {
|
||||
@@ -4012,6 +4125,46 @@ fn dispatch_data(cmd: DataCmd, env: &project::Env) {
|
||||
}
|
||||
}
|
||||
|
||||
fn dispatch_measure(cmd: MeasureCmd, env: &project::Env) {
|
||||
match cmd.command {
|
||||
MeasureCommand::Ic(a) => dispatch_measure_ic(a, env),
|
||||
}
|
||||
}
|
||||
|
||||
fn dispatch_measure_ic(a: MeasureIcCmd, env: &project::Env) {
|
||||
let traces = env.trace_store().read(&a.run).unwrap_or_else(|e| {
|
||||
eprintln!("aura: reading run '{}' traces failed: {e}", a.run);
|
||||
std::process::exit(1);
|
||||
});
|
||||
let series = |tap: &str| -> Vec<(Timestamp, f64)> {
|
||||
let ct = traces.taps.iter().find(|t| t.tap == tap).unwrap_or_else(|| {
|
||||
let have: Vec<&str> = traces.taps.iter().map(|t| t.tap.as_str()).collect();
|
||||
eprintln!("aura: run '{}' has no tap '{tap}' (taps: {have:?})", a.run);
|
||||
std::process::exit(1);
|
||||
});
|
||||
// column 0 is the tap's value series; `f64_field` (already imported in main.rs,
|
||||
// used by run_signal_r) projects field 0 of the (Timestamp, Vec<Scalar>) rows
|
||||
// ColumnarTrace::to_rows rebuilds (all cells are Scalar::f64, report.rs:248, so
|
||||
// f64_field never hits its kind-mismatch panic).
|
||||
f64_field(&ct.to_rows(), 0)
|
||||
};
|
||||
let signal = series(&a.signal);
|
||||
let price = series(&a.price);
|
||||
let out = information_coefficient(&signal, &price, a.horizon, a.permutations, a.seed);
|
||||
let report = IcReport {
|
||||
run: a.run,
|
||||
signal_tap: a.signal,
|
||||
price_tap: a.price,
|
||||
horizon: a.horizon,
|
||||
permutations: a.permutations,
|
||||
seed: a.seed,
|
||||
n_pairs: out.n_pairs,
|
||||
information_coefficient: out.information_coefficient,
|
||||
overfit_probability: out.overfit_probability,
|
||||
};
|
||||
println!("{}", report.to_json());
|
||||
}
|
||||
|
||||
/// `aura data list` (#264 cut 2): print the archive's known symbols, sorted
|
||||
/// ascending, one per line — the discovery step before `aura data coverage`
|
||||
/// or scoping a campaign's instrument matrix. Resolves the archive root
|
||||
@@ -4498,6 +4651,7 @@ fn main() {
|
||||
Command::Process(a) => research_docs::process_cmd(a, &env),
|
||||
Command::Campaign(a) => research_docs::campaign_cmd(a, &env),
|
||||
Command::Data(a) => dispatch_data(a, &env),
|
||||
Command::Measure(a) => dispatch_measure(a, &env),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6526,3 +6680,114 @@ mod tests {
|
||||
assert_eq!(stop_k, R_SMA_STOP_K, "omitted --stop-k defaults to the regime");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod ic_tests {
|
||||
use super::*;
|
||||
use aura_core::Timestamp;
|
||||
|
||||
fn ts(i: i64) -> Timestamp {
|
||||
Timestamp(i) // Timestamp is a tuple struct over epoch-ns i64 (pub field, per report.rs)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ic_engineered_signal_is_significant() {
|
||||
// price path; forward return fr[i] = (p[i+1]-p[i])/p[i]. Engineer signal_t = fr[i]
|
||||
// exactly (a hand-built OFFLINE series — a look-ahead signal is impossible in a
|
||||
// causal run, C2, so this property lives at the unit level, not E2E).
|
||||
let price: Vec<(Timestamp, f64)> = [100.0, 101.0, 100.5, 102.0, 101.0, 103.0, 102.5, 104.0]
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, &p)| (ts(i as i64), p))
|
||||
.collect();
|
||||
let signal: Vec<(Timestamp, f64)> = (0..price.len() - 1)
|
||||
.map(|i| (ts(i as i64), (price[i + 1].1 - price[i].1) / price[i].1))
|
||||
.collect();
|
||||
let out = information_coefficient(&signal, &price, 1, 1000, 0);
|
||||
assert!(out.information_coefficient > 0.99, "ic = {}", out.information_coefficient);
|
||||
assert!(out.overfit_probability < 0.05, "p = {}", out.overfit_probability);
|
||||
assert!(out.n_pairs >= 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ic_constant_signal_is_not_significant() {
|
||||
let price: Vec<(Timestamp, f64)> = [100.0, 101.0, 100.5, 102.0, 101.0, 103.0]
|
||||
.iter().enumerate().map(|(i, &p)| (ts(i as i64), p)).collect();
|
||||
let signal: Vec<(Timestamp, f64)> =
|
||||
(0..price.len()).map(|i| (ts(i as i64), 1.0)).collect(); // constant → zero variance
|
||||
let out = information_coefficient(&signal, &price, 1, 1000, 0);
|
||||
assert_eq!(out.information_coefficient, 0.0);
|
||||
assert_eq!(out.overfit_probability, 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ic_varying_uncorrelated_signal_is_not_significant() {
|
||||
// The false-positive control on a NON-degenerate null: a VARYING signal (unlike the
|
||||
// constant case above, whose null is degenerate) that is exactly uncorrelated with the
|
||||
// forward returns. price is chosen so forward returns are exactly [0.01, 0.02, 0.03,
|
||||
// 0.04]; signal is a permutation of those values arranged orthogonal to them
|
||||
// (Σ centred products = 0 → IC = 0), so the permutation null genuinely varies yet the
|
||||
// raw IC sits in its bulk (~half the permutations exceed it), not its tail → not
|
||||
// significant. This is the core guarantee of a significance test.
|
||||
let price: Vec<(Timestamp, f64)> = [100.0, 101.0, 103.02, 106.1106, 110.355024]
|
||||
.iter().enumerate().map(|(i, &p)| (ts(i as i64), p)).collect();
|
||||
let signal: Vec<(Timestamp, f64)> = [0.03, 0.01, 0.04, 0.02]
|
||||
.iter().enumerate().map(|(i, &s)| (ts(i as i64), s)).collect();
|
||||
let out = information_coefficient(&signal, &price, 1, 1000, 0);
|
||||
assert_eq!(out.n_pairs, 4);
|
||||
assert!(out.information_coefficient.abs() < 1e-6, "ic ~ 0 expected, got {}", out.information_coefficient);
|
||||
assert!(
|
||||
out.overfit_probability > 0.1,
|
||||
"an uncorrelated signal must not be significant: p = {}",
|
||||
out.overfit_probability
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ic_is_deterministic_given_seed() {
|
||||
let price: Vec<(Timestamp, f64)> = [100.0, 101.0, 100.5, 102.0, 101.0, 103.0, 102.5]
|
||||
.iter().enumerate().map(|(i, &p)| (ts(i as i64), p)).collect();
|
||||
let signal: Vec<(Timestamp, f64)> =
|
||||
[0.3, -0.1, 0.4, -0.2, 0.1, 0.5, -0.3].iter().enumerate()
|
||||
.map(|(i, &s)| (ts(i as i64), s)).collect();
|
||||
let a = information_coefficient(&signal, &price, 1, 500, 7);
|
||||
let b = information_coefficient(&signal, &price, 1, 500, 7);
|
||||
assert_eq!(a.information_coefficient, b.information_coefficient);
|
||||
assert_eq!(a.overfit_probability, b.overfit_probability);
|
||||
assert_eq!(a.n_pairs, b.n_pairs);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ic_degenerate_floor_on_too_few_pairs() {
|
||||
let price = vec![(ts(0), 100.0)]; // one row → no forward return → 0 pairs
|
||||
let signal = vec![(ts(0), 1.0)];
|
||||
let out = information_coefficient(&signal, &price, 1, 1000, 0);
|
||||
assert_eq!(out.information_coefficient, 0.0);
|
||||
assert_eq!(out.overfit_probability, 1.0);
|
||||
assert_eq!(out.n_pairs, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ic_pairs_only_on_exact_timestamp_overlap() {
|
||||
// Property: alignment is by EXACT ts match, unmatched dropped (the cross-cadence
|
||||
// case). A price ts carrying no signal is dropped, and a signal ts absent from the
|
||||
// price spine is ignored — so n_pairs is the OVERLAP count, never the price length,
|
||||
// and the dropped rows never enter the correlation.
|
||||
//
|
||||
// price spine ts 0..8 → horizon-1 forward returns exist at ts 0..7. Signal is placed
|
||||
// ONLY at ts {1,2,4,5} (leaving price ts {0,3,6} without a signal → dropped) plus two
|
||||
// decoy ts {50,60} absent from the price spine (→ never looked up). Where present, the
|
||||
// signal equals the forward return exactly, so the 4 matched pairs correlate perfectly.
|
||||
let price: Vec<(Timestamp, f64)> =
|
||||
[100.0, 101.0, 100.5, 102.0, 101.0, 103.0, 102.5, 104.0]
|
||||
.iter().enumerate().map(|(i, &p)| (ts(i as i64), p)).collect();
|
||||
let fr = |i: usize| (price[i + 1].1 - price[i].1) / price[i].1;
|
||||
let mut signal: Vec<(Timestamp, f64)> =
|
||||
[1usize, 2, 4, 5].iter().map(|&i| (ts(i as i64), fr(i))).collect();
|
||||
signal.push((ts(50), 999.0)); // decoy ts not on the price spine → unmatched, dropped
|
||||
signal.push((ts(60), -999.0));
|
||||
let out = information_coefficient(&signal, &price, 1, 1000, 0);
|
||||
assert_eq!(out.n_pairs, 4, "only the 4 overlapping ts pair; price ts 0/3/6 and the decoys drop");
|
||||
assert!(out.information_coefficient > 0.99, "ic = {}", out.information_coefficient);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user