From 0a06afed298ed63fbf864a365d0aa3505b0c7f62 Mon Sep 17 00:00:00 2001 From: Brummel Date: Fri, 26 Jun 2026 19:04:53 +0200 Subject: [PATCH] feat(0078): cross-instrument generalization as a validation step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Last piece of the Inferential-validation milestone (#146): a new `aura generalize` subcommand grades how consistently a single candidate holds across a set of instruments — the across-instrument axis of the same anti-false-discovery concern as trials-deflation (#144, within one family) and plateau-over-peak (#145, within one instrument's surface). Aggregator/validator, not a third selector (user-ratified): it runs a brought candidate (a single-cell stage1-r grid — --fast/--slow/--stop- length/--stop-k pinned to single values) across an instrument list (--real SYM1,SYM2,...), collects per-instrument R-metrics, and reduces them to a worst-case floor (min over instruments) + a sign-agreement count + the per-instrument breakdown. R-only (C10 — R is the only account/instrument-agnostic unit); the worst-case min is the cross- instrument sibling of PlateauMode::Worst and satisfies the non- domination constraint by construction (a strong instrument cannot lift a min). Rejected: a cross-instrument t-stat mean/std*sqrt(M), which reintroduces the pooled mean the milestone warns against. - RunManifest gains a first-class `instrument` lineage field (serde- widened, C14/C18 — legacy lines load as None, existing manifest bytes unchanged); the compat read-mirror threads it in lockstep. - FamilyKind::CrossInstrument (C12 comparison axis); the M per-instrument runs persist as a CrossInstrument family, each member self-identifying via its stamped instrument. The aggregate is recomputable from the members (McAggregate precedent), printed live. - The generalization reduction lives registry-side beside optimize_ deflated/optimize_plateau (C9), reusing resolve_metric/metric_value/ is_r_metric; check_r_metric is the data-free pre-check the CLI runs before evaluating any instrument. - Additive: existing sweep/walkforward/mc/standalone paths stay byte- identical (C23 — instrument omitted when None via skip_serializing_if). Verified by the orchestrator: cargo test --workspace green (0 failed — the new generalization_*, check_r_metric, parse_generalize_*, and the gated cross-instrument E2E all pass; the E2E ran the real GER40/USDJPY path on this host, exit 0); cargo clippy --workspace --all-targets -D warnings clean. closes #146 --- crates/aura-cli/src/main.rs | 199 +++++++++++++++++- crates/aura-cli/tests/cli_run.rs | 194 +++++++++++++++++ crates/aura-engine/src/blueprint.rs | 1 + crates/aura-engine/src/mc.rs | 2 + crates/aura-engine/src/report.rs | 41 +++- crates/aura-engine/src/sweep.rs | 1 + crates/aura-engine/src/walkforward.rs | 1 + crates/aura-engine/tests/random_sweep_e2e.rs | 1 + .../examples/ger40_breakout_sweep.rs | 1 + .../examples/ger40_breakout_walkforward.rs | 1 + .../examples/shared/breakout_real.rs | 1 + .../aura-ingest/tests/ger40_breakout_world.rs | 1 + crates/aura-ingest/tests/real_bars.rs | 1 + crates/aura-ingest/tests/streaming_seam.rs | 1 + crates/aura-registry/src/compat.rs | 5 +- crates/aura-registry/src/lib.rs | 159 +++++++++++++- crates/aura-registry/src/lineage.rs | 1 + crates/aura-registry/src/trace_store.rs | 1 + 18 files changed, 604 insertions(+), 8 deletions(-) diff --git a/crates/aura-cli/src/main.rs b/crates/aura-cli/src/main.rs index 2256403..8a7a786 100644 --- a/crates/aura-cli/src/main.rs +++ b/crates/aura-cli/src/main.rs @@ -24,9 +24,9 @@ use aura_engine::{ VecSource, WalkForwardResult, WindowBounds, WindowRoller, WindowRun, }; use aura_registry::{ - group_families, mc_member_reports, optimize_deflated, optimize_plateau, rank_by, - sweep_member_reports, walkforward_member_reports, FamilyKind, FamilyMember, NameKind, - PlateauMode, Registry, RunTraces, TraceStore, WriteKind, + check_r_metric, generalization, group_families, mc_member_reports, optimize_deflated, + optimize_plateau, rank_by, sweep_member_reports, walkforward_member_reports, FamilyKind, + FamilyMember, Generalization, NameKind, PlateauMode, Registry, RunTraces, TraceStore, WriteKind, }; use aura_std::{ Add, Bias, Delay, Ema, GatedRecorder, Gt, Latch, LinComb, LongOnly, Mul, Recorder, RollingMax, @@ -174,6 +174,7 @@ fn sim_optimal_manifest( seed, broker: format!("sim-optimal(pip_size={pip_size})"), selection: None, + instrument: None, } } @@ -1662,6 +1663,69 @@ fn parse_sweep_args( Ok((strategy, name, persist, real.finish(&usage)?, grid)) } +/// Parse the `generalize` tail: +/// `[--strategy stage1-r] --real --fast --slow --stop-length +/// --stop-k [--from ] [--to ] [--metric ] [--name ]`. +/// The candidate is a single cell: each grid knob takes exactly one value and all +/// four are required. `--real` is a comma list of >= 2 distinct non-empty symbols. +/// Pure (no I/O / exit) so the grammar is unit-testable. +#[allow(clippy::type_complexity)] +fn parse_generalize_args( + rest: &[&str], +) -> Result<(String, Vec, Stage1RGrid, String, Option, Option), String> { + let usage = || "generalize [--strategy stage1-r] --real --fast --slow --stop-length --stop-k [--from ] [--to ] [--metric ] [--name ]".to_string(); + let mut symbols: Option> = None; + let mut from_ms: Option = None; + let mut to_ms: Option = None; + let mut fast: Option = None; + let mut slow: Option = None; + let mut stop_length: Option = None; + let mut stop_k: Option = None; + let mut metric = "expectancy_r".to_string(); + let mut name = "generalize".to_string(); + // a single grid knob, required and single-valued (csv with exactly one item) + let one = |value: &str, usage: &dyn Fn() -> String| -> Result<(), String> { + let items: Vec<&str> = value.split(',').collect(); + if items.len() != 1 || items[0].is_empty() { return Err(usage()); } + Ok(()) + }; + let mut tail = rest; + while let Some((flag, t)) = tail.split_first() { + let (value, t) = t.split_first().ok_or_else(usage)?; + match *flag { + "--strategy" => { if *value != "stage1-r" { return Err(usage()); } } + "--real" => { + let parts: Vec = value.split(',').map(|s| s.to_string()).collect(); + if parts.iter().any(|s| s.is_empty()) { return Err(usage()); } + symbols = Some(parts); + } + "--from" => from_ms = Some(value.parse().map_err(|_| usage())?), + "--to" => to_ms = Some(value.parse().map_err(|_| usage())?), + "--fast" => { one(value, &usage)?; fast = Some(value.parse().map_err(|_| usage())?); } + "--slow" => { one(value, &usage)?; slow = Some(value.parse().map_err(|_| usage())?); } + "--stop-length" => { one(value, &usage)?; stop_length = Some(value.parse().map_err(|_| usage())?); } + "--stop-k" => { one(value, &usage)?; stop_k = Some(value.parse().map_err(|_| usage())?); } + "--metric" => metric = (*value).to_string(), + "--name" => name = (*value).to_string(), + _ => return Err(usage()), + } + tail = t; + } + let symbols = symbols.ok_or_else(usage)?; + if symbols.len() < 2 { return Err(usage()); } + // distinct: a duplicate would double-count one instrument in the floor / sign count + let mut seen = std::collections::HashSet::new(); + if !symbols.iter().all(|s| seen.insert(s.clone())) { return Err(usage()); } + let grid = Stage1RGrid { + fast: vec![fast.ok_or_else(usage)?], + slow: vec![slow.ok_or_else(usage)?], + stop_length: vec![stop_length.ok_or_else(usage)?], + stop_k: vec![stop_k.ok_or_else(usage)?], + ..Stage1RGrid::default() + }; + Ok((name, symbols, grid, metric, from_ms, to_ms)) +} + /// Parse the `walkforward` tail: /// `[--strategy ] [--real [--from ] [--to ]] [--name | --trace ] [--fast ] [--slow ] [--stop-length ] [--stop-k ] [--select ]`. /// Defaults: SMA-cross, synthetic, name "walkforward", no persist. `--name`/`--trace` @@ -1767,6 +1831,48 @@ fn run_sweep(strategy: Strategy, name: &str, persist: bool, data: DataSource, gr } } +/// `aura generalize --real --fast --slow --stop-length +/// --stop-k `: grade one stage1-r candidate (a single-cell grid) across an +/// instrument list. Pre-checks the R-metric data-free (refuse a non-R / unknown +/// metric before any run), runs the candidate per instrument (stamping each report's +/// manifest `instrument`), reduces to the worst-case floor + sign-agreement + +/// per-instrument breakdown (`generalization`, C9/C10), prints the aggregate, and +/// persists the per-instrument members as a `CrossInstrument` family (C12/C18). +fn run_generalize( + name: &str, + symbols: &[String], + grid: &Stage1RGrid, + metric: &str, + from_ms: Option, + to_ms: Option, +) { + // data-free metric pre-check: refuse a non-R / unknown metric before any run. + if let Err(e) = check_r_metric(metric) { + eprintln!("aura: {e}"); + std::process::exit(2); + } + let mut members: Vec = Vec::new(); + for symbol in symbols { + let choice = DataChoice::Real { symbol: symbol.clone(), from_ms, to_ms }; + let data = DataSource::from_choice(choice); // per-instrument pip_or_refuse + has_symbol + let family = stage1_r_sweep_family(None, &data, grid); // single-cell grid -> 1 member + let mut report = family.points[0].report.clone(); + report.manifest.instrument = Some(symbol.clone()); + members.push(report); + } + let pairs: Vec<(String, &RunReport)> = symbols.iter().cloned().zip(members.iter()).collect(); + let agg = match generalization(&pairs, metric) { + Ok(a) => a, + Err(e) => { eprintln!("aura: {e}"); std::process::exit(2); } + }; + println!("{}", generalize_json(&agg)); + let reg = default_registry(); + match reg.append_family(name, FamilyKind::CrossInstrument, &members) { + Ok(id) => println!("{{\"family_id\":\"{id}\"}}"), + Err(e) => { eprintln!("aura: failed to persist family: {e}"); std::process::exit(1); } + } +} + /// `aura walkforward [--name |--trace ]`: run a built-in rolling walk-forward /// over the sample blueprint + a synthetic windowed source. Per window: sweep the /// built-in grid on the in-sample slice, optimize by total_pips (axis 2 inside axis 3, @@ -1983,6 +2089,25 @@ fn walkforward_summary_json(result: &WalkForwardResult) -> String { serde_json::json!({ "walkforward": obj }).to_string() } +/// The cross-instrument generalization line: the chosen metric, instrument count, +/// worst-case floor, sign-agreement count, and the per-instrument breakdown. Canonical +/// JSON (C14), mirroring `walkforward_summary_json`'s `{"generalize": obj}` shape. +fn generalize_json(agg: &Generalization) -> String { + let per: Vec = agg + .per_instrument + .iter() + .map(|(sym, v)| serde_json::json!([sym, v])) + .collect(); + let obj = serde_json::json!({ + "metric": agg.selection_metric, + "n_instruments": agg.n_instruments, + "worst_case": agg.worst_case, + "sign_agreement": agg.sign_agreement, + "per_instrument": per, + }); + serde_json::json!({ "generalize": obj }).to_string() +} + /// A longer deterministic stream than `showcase_prices` — enough for several /// IS/OOS windows with SMA warm-up. Seed-determined via `SyntheticSpec` (C1). fn walkforward_prices() -> Vec<(Timestamp, Scalar)> { @@ -2954,7 +3079,7 @@ fn run_dispatch(args: RunArgs) -> Result { } const USAGE: &str = - "usage: aura run [--harness ] [--real [--from ] [--to ]] [--trace ] | aura chart [--tap ] [--panels] | aura graph | aura sweep [--strategy ] [--real [--from ] [--to ]] [--name |--trace ] | aura mc [--name |--trace ] | aura mc --strategy stage1-r [--real [--from ] [--to ]] [--fast ] [--slow ] [--stop-length ] [--stop-k ] [--block-len ] [--resamples ] [--seed ] | aura walkforward [--strategy ] [--real [--from ] [--to ]] [--name |--trace ] [--fast ] [--slow ] [--stop-length ] [--stop-k ] | aura runs families | aura runs family [rank ]"; + "usage: aura run [--harness ] [--real [--from ] [--to ]] [--trace ] | aura chart [--tap ] [--panels] | aura graph | aura sweep [--strategy ] [--real [--from ] [--to ]] [--name |--trace ] | aura mc [--name |--trace ] | aura mc --strategy stage1-r [--real [--from ] [--to ]] [--fast ] [--slow ] [--stop-length ] [--stop-k ] [--block-len ] [--resamples ] [--seed ] | aura walkforward [--strategy ] [--real [--from ] [--to ]] [--name |--trace ] [--fast ] [--slow ] [--stop-length ] [--stop-k ] | aura generalize [--strategy stage1-r] --real --fast --slow --stop-length --stop-k [--from ] [--to ] [--metric ] [--name ] | aura runs families | aura runs family [rank ]"; fn main() { // Collect argv and match the whole vector: every accepted form is exhaustive, @@ -3008,6 +3133,15 @@ fn main() { std::process::exit(2); } }, + ["generalize", rest @ ..] => match parse_generalize_args(rest) { + Ok((name, symbols, grid, metric, from_ms, to_ms)) => { + run_generalize(&name, &symbols, &grid, &metric, from_ms, to_ms) + } + Err(msg) => { + eprintln!("aura: {msg}"); + std::process::exit(2); + } + }, ["mc", rest @ ..] => match parse_mc_args(rest) { Ok(McArgs::Synthetic { name, persist }) => run_mc(&name, persist), Ok(McArgs::RealR { choice, grid, block_len, n_resamples, seed }) => { @@ -3952,6 +4086,63 @@ mod tests { assert!(parse_sweep_args(&["--channel", ""]).is_err()); // empty item } + #[test] + fn parse_generalize_requires_a_single_value_candidate() { + // a candidate is one cell: a multi-value grid flag is refused + assert!(parse_generalize_args(&[ + "--real", "GER40,USDJPY", "--fast", "2,3", "--slow", "12", + "--stop-length", "14", "--stop-k", "2.0", + ]).is_err()); + } + + #[test] + fn parse_generalize_requires_all_four_knobs() { + // --slow absent -> refuse (no defaulting to the multi-value sweep grid) + assert!(parse_generalize_args(&[ + "--real", "GER40,USDJPY", "--fast", "3", + "--stop-length", "14", "--stop-k", "2.0", + ]).is_err()); + } + + #[test] + fn parse_generalize_refuses_fewer_than_two_instruments() { + assert!(parse_generalize_args(&[ + "--real", "GER40", "--fast", "3", "--slow", "12", + "--stop-length", "14", "--stop-k", "2.0", + ]).is_err()); + } + + #[test] + fn parse_generalize_refuses_a_duplicate_instrument() { + assert!(parse_generalize_args(&[ + "--real", "GER40,GER40", "--fast", "3", "--slow", "12", + "--stop-length", "14", "--stop-k", "2.0", + ]).is_err()); + } + + #[test] + fn parse_generalize_refuses_a_non_stage1r_strategy() { + assert!(parse_generalize_args(&[ + "--strategy", "sma", "--real", "GER40,USDJPY", "--fast", "3", "--slow", "12", + "--stop-length", "14", "--stop-k", "2.0", + ]).is_err()); + } + + #[test] + fn parse_generalize_defaults_name_and_metric() { + let (name, symbols, grid, metric, _from, _to) = parse_generalize_args(&[ + "--real", "GER40,USDJPY", "--fast", "3", "--slow", "12", + "--stop-length", "14", "--stop-k", "2.0", + ]).expect("valid generalize args"); + assert_eq!(name, "generalize"); + assert_eq!(metric, "expectancy_r"); + assert_eq!(symbols, vec!["GER40".to_string(), "USDJPY".to_string()]); + assert_eq!(grid.fast, vec![3]); + assert_eq!(grid.slow, vec![12]); + assert_eq!(grid.stop_length, vec![14]); + assert_eq!(grid.stop_k, vec![2.0]); + } + /// Property: the meanrev `--window` / `--band-k` flags parse comma-separated /// lists onto `Stage1RGrid.{window,band_k}` (the meanrev family's signal axes), /// with the same strictness as the other grid flags — absent flags keep the diff --git a/crates/aura-cli/tests/cli_run.rs b/crates/aura-cli/tests/cli_run.rs index 6871c28..2ac1bd2 100644 --- a/crates/aura-cli/tests/cli_run.rs +++ b/crates/aura-cli/tests/cli_run.rs @@ -2670,3 +2670,197 @@ fn mc_stage1_r_oversized_block_len_clamps_and_collapses() { assert!((p5 - p95).abs() < 1e-12, "single-block bootstrap must have zero spread: p5={p5} p95={p95}"); let _ = std::fs::remove_dir_all(&dir); } + +/// Property: `aura generalize` grades a single stage1-r candidate across two +/// instruments — its stdout carries the per-instrument breakdown + the worst-case +/// floor + the sign-agreement, then the persisted CrossInstrument family handle. +/// Gated on local GER40/USDJPY data (the shared Sept-2024 window), skips cleanly +/// when the archive is absent. +#[test] +fn generalize_grades_a_candidate_across_two_instruments() { + const FROM_MS: &str = "1725148800000"; + const TO_MS: &str = "1727740799999"; + let cwd = temp_cwd("generalize-two"); + let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args([ + "generalize", "--strategy", "stage1-r", "--real", "GER40,USDJPY", + "--fast", "3", "--slow", "12", "--stop-length", "14", "--stop-k", "2.0", + "--from", FROM_MS, "--to", TO_MS, + ]) + .current_dir(&cwd) + .output() + .expect("spawn aura"); + if out.status.code() == Some(2) { + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("no local data") || stderr.contains("no recorded geometry"), + "exit 2 must be a data refusal, got: {stderr}" + ); + eprintln!("skip: no local GER40/USDJPY data"); + return; + } + assert_eq!(out.status.code(), Some(0), "exit: {:?}", out.status); + let stdout = String::from_utf8(out.stdout).expect("utf-8 stdout"); + assert!(stdout.contains("\"generalize\":"), "aggregate object: {stdout}"); + assert!(stdout.contains("\"n_instruments\":2"), "two instruments: {stdout}"); + assert!(stdout.contains("\"worst_case\":"), "worst_case present: {stdout}"); + assert!(stdout.contains("\"sign_agreement\":"), "sign_agreement present: {stdout}"); + assert!(stdout.contains("GER40") && stdout.contains("USDJPY"), "per-instrument breakdown: {stdout}"); + assert!(stdout.contains("\"family_id\":\"generalize-0\""), "family handle: {stdout}"); +} + +/// Property: a single-instrument generalize is refused at parse time (exit 2), +/// before any data access — so it asserts the arity refusal on any machine. +#[test] +fn generalize_refuses_a_single_instrument() { + let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args([ + "generalize", "--strategy", "stage1-r", "--real", "GER40", + "--fast", "3", "--slow", "12", "--stop-length", "14", "--stop-k", "2.0", + ]) + .output() + .expect("spawn aura"); + assert_eq!(out.status.code(), Some(2), "single instrument must exit 2"); +} + +/// Property: a non-R metric is refused before any instrument runs (the data-free +/// `check_r_metric` pre-check), so this asserts the R-only refusal on any machine. +#[test] +fn generalize_refuses_a_non_r_metric() { + let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args([ + "generalize", "--strategy", "stage1-r", "--real", "GER40,USDJPY", + "--fast", "3", "--slow", "12", "--stop-length", "14", "--stop-k", "2.0", + "--metric", "total_pips", + ]) + .output() + .expect("spawn aura"); + assert_eq!(out.status.code(), Some(2), "a non-R metric must exit 2"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(stderr.contains("R-only"), "must name the R-only refusal, got: {stderr}"); +} + +/// Property: a duplicate instrument in the `--real` list is refused at the binary +/// boundary (exit 2, before any data access) — `GER40,GER40` never runs. The spec's +/// double-count hazard: a repeated symbol would be evaluated twice and so counted +/// twice in both the worst-case floor and the sign-agreement count, silently +/// corrupting the generalization grade. The parser unit test pins the grammar; this +/// pins that the parse error actually reaches `exit(2)` through the dispatch arm +/// (a swallowed Err would let a duplicated run proceed). Data-free — the duplicate +/// check precedes any geometry/archive lookup, so it asserts on any machine. +#[test] +fn generalize_refuses_a_duplicate_instrument() { + let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args([ + "generalize", "--strategy", "stage1-r", "--real", "GER40,GER40", + "--fast", "3", "--slow", "12", "--stop-length", "14", "--stop-k", "2.0", + ]) + .output() + .expect("spawn aura"); + assert_eq!(out.status.code(), Some(2), "a duplicate instrument must exit 2"); + assert!( + out.stdout.is_empty(), + "the refusal must not leak an aggregate to stdout, got: {:?}", + String::from_utf8_lossy(&out.stdout) + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(stderr.contains("generalize"), "must surface the generalize usage, got: {stderr}"); +} + +/// Property: a non-`stage1-r` strategy is refused at the binary boundary (exit 2, +/// no run, data-free). The candidate must produce R (C10) — the cross-instrument +/// reduction is R-only — so only the stage1-r grid is an admissible candidate. The +/// parser unit test pins the grammar refusal; this pins that `--strategy sma` +/// reaches `exit(2)` through the dispatch arm rather than silently defaulting to a +/// running candidate. Asserts on any machine (the strategy check precedes any run). +#[test] +fn generalize_refuses_a_non_stage1r_strategy() { + let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args([ + "generalize", "--strategy", "sma", "--real", "GER40,USDJPY", + "--fast", "3", "--slow", "12", "--stop-length", "14", "--stop-k", "2.0", + ]) + .output() + .expect("spawn aura"); + assert_eq!(out.status.code(), Some(2), "a non-stage1-r strategy must exit 2"); + assert!( + out.stdout.is_empty(), + "the refusal must not leak an aggregate to stdout, got: {:?}", + String::from_utf8_lossy(&out.stdout) + ); +} + +/// Property: a multi-value candidate flag is refused at the binary boundary (exit 2, +/// no run, data-free). A *candidate* is a single grid cell, not a sweep — so any of +/// `--fast/--slow/--stop-length/--stop-k` carrying more than one value (`--fast 2,3`) +/// is refused, never silently widened into a multi-cell run whose generalization +/// grade would be ill-defined (the reduction grades one candidate, not a family). +/// The parser unit test pins the grammar; this pins the wiring to `exit(2)`. +#[test] +fn generalize_refuses_a_multi_value_candidate_flag() { + let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args([ + "generalize", "--strategy", "stage1-r", "--real", "GER40,USDJPY", + "--fast", "2,3", "--slow", "12", "--stop-length", "14", "--stop-k", "2.0", + ]) + .output() + .expect("spawn aura"); + assert_eq!(out.status.code(), Some(2), "a multi-value candidate flag must exit 2"); + assert!( + out.stdout.is_empty(), + "the refusal must not leak an aggregate to stdout, got: {:?}", + String::from_utf8_lossy(&out.stdout) + ); +} + +/// Property: a successful `aura generalize` persists its per-instrument runs as a +/// discoverable `CrossInstrument` family — the C12 comparison axis / C18 lineage at +/// the built-binary boundary. The unit `cli_families_persist_and_round_trip_per_kind` +/// covers Sweep/MonteCarlo/WalkForward but not CrossInstrument; this is the +/// cross-instrument sibling of `mc_runs_persists_a_monte_carlo_family_and_lists_it`: +/// after the run, `aura runs families` lists `generalize-0` with +/// `"kind":"CrossInstrument"` and `"members":2` (one member per instrument). A +/// regression that mis-tagged the family kind, or persisted the wrong member count, +/// would slip past the stdout-only happy-path assertion above but fail here. Gated on +/// local GER40/USDJPY data (the shared Sept-2024 window); skips cleanly when absent. +#[test] +fn generalize_persists_a_discoverable_cross_instrument_family() { + const FROM_MS: &str = "1725148800000"; + const TO_MS: &str = "1727740799999"; + let cwd = temp_cwd("generalize-family"); + let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args([ + "generalize", "--strategy", "stage1-r", "--real", "GER40,USDJPY", + "--fast", "3", "--slow", "12", "--stop-length", "14", "--stop-k", "2.0", + "--from", FROM_MS, "--to", TO_MS, + ]) + .current_dir(&cwd) + .output() + .expect("spawn aura"); + if out.status.code() == Some(2) { + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("no local data") || stderr.contains("no recorded geometry"), + "exit 2 must be a data refusal, got: {stderr}" + ); + eprintln!("skip: no local GER40/USDJPY data"); + let _ = std::fs::remove_dir_all(&cwd); + return; + } + assert_eq!(out.status.code(), Some(0), "exit: {:?}", out.status); + + // the run persisted the per-instrument members as a discoverable CrossInstrument family. + let fams = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args(["runs", "families"]) + .current_dir(&cwd) + .output() + .expect("spawn families"); + assert!(fams.status.success(), "families exit: {:?}", fams.status); + let fams_out = String::from_utf8(fams.stdout).expect("utf-8"); + assert_eq!(fams_out.lines().count(), 1, "families: {fams_out:?}"); + assert!(fams_out.contains("\"family_id\":\"generalize-0\""), "families: {fams_out:?}"); + assert!(fams_out.contains("\"kind\":\"CrossInstrument\""), "families: {fams_out:?}"); + assert!(fams_out.contains("\"members\":2"), "one member per instrument: {fams_out:?}"); + + let _ = std::fs::remove_dir_all(&cwd); +} diff --git a/crates/aura-engine/src/blueprint.rs b/crates/aura-engine/src/blueprint.rs index 83351c1..4e3ca49 100644 --- a/crates/aura-engine/src/blueprint.rs +++ b/crates/aura-engine/src/blueprint.rs @@ -946,6 +946,7 @@ mod tests { seed: 0, broker: "test".to_string(), selection: None, + instrument: None, }, metrics: summarize(&equity, &exposure), } diff --git a/crates/aura-engine/src/mc.rs b/crates/aura-engine/src/mc.rs index 3fc5a81..9d0af81 100644 --- a/crates/aura-engine/src/mc.rs +++ b/crates/aura-engine/src/mc.rs @@ -243,6 +243,7 @@ mod tests { seed, broker: "test".to_string(), selection: None, + instrument: None, }, metrics: summarize(&equity, &exposure), } @@ -267,6 +268,7 @@ mod tests { seed: i as u64, broker: "t".to_string(), selection: None, + instrument: None, }, metrics: RunMetrics { total_pips: v, diff --git a/crates/aura-engine/src/report.rs b/crates/aura-engine/src/report.rs index bbd0fbe..cfb4e6a 100644 --- a/crates/aura-engine/src/report.rs +++ b/crates/aura-engine/src/report.rs @@ -479,6 +479,12 @@ pub struct RunManifest { /// serde widening (C14/C18), identical idiom to `RunMetrics.r`. #[serde(default, skip_serializing_if = "Option::is_none")] pub selection: Option, + /// The instrument this run evaluated, when it is a real-data run; `None` for a + /// synthetic run and for every pre-0078 line. The "instrument set as lineage" + /// (C18) for a cross-instrument family is each member's stamped symbol. Same + /// one-directional serde widening as `selection` (C14/C18). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub instrument: Option, } /// A run's full structured result: the descriptor plus the metrics it @@ -770,11 +776,38 @@ mod tests { fn runmanifest_without_selection_serialises_without_the_key() { let m = RunManifest { commit: "c".into(), params: vec![], window: (Timestamp(0), Timestamp(0)), - seed: 0, broker: "b".into(), selection: None, + seed: 0, broker: "b".into(), selection: None, instrument: None, }; assert!(!serde_json::to_string(&m).unwrap().contains("selection")); } + #[test] + fn runmanifest_without_instrument_field_deserialises_to_none() { + // A pre-0078 line carries no `instrument` key; serde `default` -> None. + let json = r#"{"commit":"c","params":[],"window":[0,0],"seed":0,"broker":"b"}"#; + let m: RunManifest = serde_json::from_str(json).expect("legacy manifest loads"); + assert!(m.instrument.is_none()); + } + + #[test] + fn runmanifest_instrument_round_trips_and_omits_when_none() { + let with = RunManifest { + commit: "c".into(), params: vec![], window: (Timestamp(0), Timestamp(0)), + seed: 0, broker: "b".into(), selection: None, instrument: Some("GER40".into()), + }; + let json = serde_json::to_string(&with).unwrap(); + assert!(json.contains("\"instrument\":\"GER40\""), "json: {json}"); + let back: RunManifest = serde_json::from_str(&json).unwrap(); + assert_eq!(back.instrument, Some("GER40".to_string())); + + let without = RunManifest { + commit: "c".into(), params: vec![], window: (Timestamp(0), Timestamp(0)), + seed: 0, broker: "b".into(), selection: None, instrument: None, + }; + // skip_serializing_if omits the key when None -> existing manifest bytes unchanged (C23). + assert!(!serde_json::to_string(&without).unwrap().contains("instrument")); + } + #[test] fn family_selection_round_trips_on_the_manifest() { let sel = FamilySelection { @@ -786,7 +819,7 @@ mod tests { }; let m = RunManifest { commit: "c".into(), params: vec![], window: (Timestamp(0), Timestamp(0)), - seed: 0, broker: "b".into(), selection: Some(sel.clone()), + seed: 0, broker: "b".into(), selection: Some(sel.clone()), instrument: None, }; let back: RunManifest = serde_json::from_str(&serde_json::to_string(&m).unwrap()).unwrap(); assert_eq!(back.selection, Some(sel)); @@ -976,6 +1009,7 @@ mod tests { seed: 0, broker: "sim-optimal(pip_size=0.0001)".to_string(), selection: None, + instrument: None, }, metrics, } @@ -1352,6 +1386,7 @@ mod tests { seed: 0, broker: "sim-optimal(pip_size=1.0)".to_string(), selection: None, + instrument: None, }, metrics: RunMetrics { total_pips: 12.0, @@ -1381,6 +1416,7 @@ mod tests { seed: 0, broker: "sim-optimal(pip_size=1.0)".to_string(), selection: None, + instrument: None, }, metrics: RunMetrics { total_pips: 12.0, max_drawdown: 1.0, bias_sign_flips: 1, r: None }, }; @@ -1402,6 +1438,7 @@ mod tests { seed: 0, broker: "sim-optimal(pip_size=1.0)".to_string(), selection: None, + instrument: None, }, metrics: RunMetrics { total_pips: 12.0, max_drawdown: 1.0, bias_sign_flips: 1, r: None }, }; diff --git a/crates/aura-engine/src/sweep.rs b/crates/aura-engine/src/sweep.rs index 6840d93..ccc06a8 100644 --- a/crates/aura-engine/src/sweep.rs +++ b/crates/aura-engine/src/sweep.rs @@ -668,6 +668,7 @@ mod tests { seed: 0, broker: "test".to_string(), selection: None, + instrument: None, }, metrics: summarize(&equity, &exposure), } diff --git a/crates/aura-engine/src/walkforward.rs b/crates/aura-engine/src/walkforward.rs index 52cf559..265de2d 100644 --- a/crates/aura-engine/src/walkforward.rs +++ b/crates/aura-engine/src/walkforward.rs @@ -292,6 +292,7 @@ mod tests { seed: 0, broker: "t".to_string(), selection: None, + instrument: None, }, metrics: summarize(&[], &[]), } diff --git a/crates/aura-engine/tests/random_sweep_e2e.rs b/crates/aura-engine/tests/random_sweep_e2e.rs index 1f596b3..072e026 100644 --- a/crates/aura-engine/tests/random_sweep_e2e.rs +++ b/crates/aura-engine/tests/random_sweep_e2e.rs @@ -116,6 +116,7 @@ fn run_point(point: &[Cell]) -> RunReport { seed: 0, broker: "test".to_string(), selection: None, + instrument: None, }, metrics: summarize(&equity, &exposure), } diff --git a/crates/aura-ingest/examples/ger40_breakout_sweep.rs b/crates/aura-ingest/examples/ger40_breakout_sweep.rs index d70bf60..4325546 100644 --- a/crates/aura-ingest/examples/ger40_breakout_sweep.rs +++ b/crates/aura-ingest/examples/ger40_breakout_sweep.rs @@ -67,6 +67,7 @@ fn run_point(server: &Arc, point: &[Cell], from: Timestamp, to: Time seed: 0, broker: "sim-optimal(pip_size=1)".to_string(), selection: None, + instrument: None, }, metrics: summarize(&equity, &exposure), } diff --git a/crates/aura-ingest/examples/ger40_breakout_walkforward.rs b/crates/aura-ingest/examples/ger40_breakout_walkforward.rs index 925dfc5..121d210 100644 --- a/crates/aura-ingest/examples/ger40_breakout_walkforward.rs +++ b/crates/aura-ingest/examples/ger40_breakout_walkforward.rs @@ -77,6 +77,7 @@ fn run_point( seed: 0, broker: "sim-optimal(pip_size=1)".to_string(), selection: None, + instrument: None, }, metrics: summarize(&equity, &exposure), }; diff --git a/crates/aura-ingest/examples/shared/breakout_real.rs b/crates/aura-ingest/examples/shared/breakout_real.rs index 94c7b44..d4c97fa 100644 --- a/crates/aura-ingest/examples/shared/breakout_real.rs +++ b/crates/aura-ingest/examples/shared/breakout_real.rs @@ -400,6 +400,7 @@ pub fn report_from_trace(trace: &[BarTrace], from: Timestamp, to: Timestamp) -> seed: 0, broker: "sim-optimal(pip_size=1)".to_string(), selection: None, + instrument: None, }, metrics, } diff --git a/crates/aura-ingest/tests/ger40_breakout_world.rs b/crates/aura-ingest/tests/ger40_breakout_world.rs index f5c6a15..4afd985 100644 --- a/crates/aura-ingest/tests/ger40_breakout_world.rs +++ b/crates/aura-ingest/tests/ger40_breakout_world.rs @@ -69,6 +69,7 @@ fn run_point(server: &Arc, point: &[Cell], from: Timestamp, to: Time seed: 0, broker: "sim-optimal(pip_size=1)".to_string(), selection: None, + instrument: None, }, metrics: summarize(&equity, &exposure), } diff --git a/crates/aura-ingest/tests/real_bars.rs b/crates/aura-ingest/tests/real_bars.rs index d58d4f5..7ebd238 100644 --- a/crates/aura-ingest/tests/real_bars.rs +++ b/crates/aura-ingest/tests/real_bars.rs @@ -90,6 +90,7 @@ fn run_sample_over(prices: Vec<(Timestamp, Scalar)>) -> RunReport { seed: 0, broker: "sim-optimal(pip_size=0.0001)".to_string(), selection: None, + instrument: None, }, metrics, } diff --git a/crates/aura-ingest/tests/streaming_seam.rs b/crates/aura-ingest/tests/streaming_seam.rs index 2977b9e..6a0bd19 100644 --- a/crates/aura-ingest/tests/streaming_seam.rs +++ b/crates/aura-ingest/tests/streaming_seam.rs @@ -90,6 +90,7 @@ fn run_sample(window: (Timestamp, Timestamp), source: Box) -> RunRep seed: 0, broker: "sim-optimal(pip_size=0.0001)".to_string(), selection: None, + instrument: None, }, metrics: summarize(&equity, &exposure), } diff --git a/crates/aura-registry/src/compat.rs b/crates/aura-registry/src/compat.rs index 4926172..d43c1b3 100644 --- a/crates/aura-registry/src/compat.rs +++ b/crates/aura-registry/src/compat.rs @@ -32,6 +32,8 @@ struct RunManifestRead { broker: String, #[serde(default)] selection: Option, + #[serde(default)] + instrument: Option, } /// A param value that accepts BOTH the current tagged [`Scalar`] form @@ -64,7 +66,7 @@ impl<'de> Deserialize<'de> for ScalarRead { impl From for RunReport { fn from(r: RunReportRead) -> Self { - let RunManifestRead { commit, params, window, seed, broker, selection } = r.manifest; + let RunManifestRead { commit, params, window, seed, broker, selection, instrument } = r.manifest; RunReport { manifest: RunManifest { commit, @@ -73,6 +75,7 @@ impl From for RunReport { seed, broker, selection, + instrument, }, metrics: r.metrics, } diff --git a/crates/aura-registry/src/lib.rs b/crates/aura-registry/src/lib.rs index cffb409..6fe1c18 100644 --- a/crates/aura-registry/src/lib.rs +++ b/crates/aura-registry/src/lib.rs @@ -428,6 +428,65 @@ pub fn optimize_deflated( })) } +/// The cross-instrument generalization aggregate (#146): a brought candidate's +/// per-instrument R-metric, reduced to a worst-case floor, a sign-agreement count, +/// and the full breakdown. R-only (C10): `total_pips` is not comparable across +/// instruments. Pure, deterministic (C1) — a fold over `per_instrument` in input +/// order. `worst_case` is `min_i metric_i`; since every R-metric is +/// higher-is-better, the min is unconditionally the conservative floor (no +/// direction branch, unlike `PlateauMode::Worst`). +#[derive(Clone, Debug, PartialEq)] +pub struct Generalization { + pub selection_metric: String, + pub n_instruments: usize, + pub worst_case: f64, + pub sign_agreement: usize, + pub per_instrument: Vec<(String, f64)>, +} + +/// Validate that `metric` is a known, R-based ranking key — the data-free pre-check +/// the CLI runs before evaluating any instrument. The registry owns metric truth +/// (C9), so the CLI never duplicates the R-set. +pub fn check_r_metric(metric: &str) -> Result<(), RegistryError> { + let m = resolve_metric(metric)?; + if is_r_metric(m) { + Ok(()) + } else { + Err(RegistryError::NonRMetric(metric.to_string())) + } +} + +/// Grade how consistently one candidate holds across instruments: read the chosen +/// R-metric from each per-instrument report and reduce to the worst-case floor + +/// the sign-agreement count + the per-instrument breakdown. R-only and `>= 2` +/// instruments (the metric check precedes the arity check so a bad metric is +/// reported even with an empty slice). Additive — reads, never re-ranks (C23). +pub fn generalization( + per_instrument: &[(String, &RunReport)], + metric: &str, +) -> Result { + let m = resolve_metric(metric)?; + if !is_r_metric(m) { + return Err(RegistryError::NonRMetric(metric.to_string())); + } + if per_instrument.len() < 2 { + return Err(RegistryError::TooFewInstruments(per_instrument.len())); + } + let vals: Vec<(String, f64)> = per_instrument + .iter() + .map(|(label, rep)| (label.clone(), metric_value(rep, m))) + .collect(); + let worst_case = vals.iter().map(|(_, v)| *v).fold(f64::INFINITY, f64::min); + let sign_agreement = vals.iter().filter(|(_, v)| *v > 0.0).count(); + Ok(Generalization { + selection_metric: metric.to_string(), + n_instruments: vals.len(), + worst_case, + sign_agreement, + per_instrument: vals, + }) +} + /// What can go wrong reading or ranking the registry. #[derive(Debug)] pub enum RegistryError { @@ -437,6 +496,11 @@ pub enum RegistryError { Parse { line: usize, source: serde_json::Error }, /// `rank_by` was given a metric name it does not know. UnknownMetric(String), + /// A known metric that is not R-based — cross-instrument generalization is + /// R-only (R is the only account/instrument-agnostic unit, C10). + NonRMetric(String), + /// A generalization was asked for with fewer than two instruments. + TooFewInstruments(usize), } impl fmt::Display for RegistryError { @@ -450,6 +514,14 @@ impl fmt::Display for RegistryError { f, "unknown metric '{m}' (known: total_pips, max_drawdown, bias_sign_flips, sqn, sqn_normalized, expectancy_r, net_expectancy_r)" ), + RegistryError::NonRMetric(m) => write!( + f, + "metric '{m}' is not comparable across instruments; cross-instrument scoring is R-only (sqn, sqn_normalized, expectancy_r, net_expectancy_r)" + ), + RegistryError::TooFewInstruments(n) => write!( + f, + "a generalization needs >= 2 instruments, got {n}" + ), } } } @@ -478,6 +550,7 @@ mod tests { seed: 0, broker: "b".to_string(), selection: None, + instrument: None, }, metrics: RunMetrics { total_pips, max_drawdown, bias_sign_flips: flips, r: None }, } @@ -812,7 +885,7 @@ mod tests { report: RunReport { manifest: RunManifest { commit: "c".into(), params: vec![], window: (Timestamp(0), Timestamp(0)), - seed: 0, broker: "b".into(), selection: None, + seed: 0, broker: "b".into(), selection: None, instrument: None, }, metrics: RunMetrics { total_pips, max_drawdown: 0.0, bias_sign_flips: 0, @@ -1095,4 +1168,88 @@ mod tests { "the plateau interior's worst neighbour is below the dip's 60.0 shoulder", ); } + + #[test] + fn generalization_worst_case_is_the_floor_not_the_mean() { + let a = report_with_r(0.0, 0.40, 0.0); // +0.40 + let b = report_with_r(0.0, -0.20, 0.0); // -0.20 (the floor) + let c = report_with_r(0.0, 0.10, 0.0); // +0.10 + let pairs = vec![("A".to_string(), &a), ("B".to_string(), &b), ("C".to_string(), &c)]; + let g = generalization(&pairs, "expectancy_r").expect("R metric"); + assert_eq!(g.n_instruments, 3); + assert!((g.worst_case - (-0.20)).abs() < 1e-12, "worst_case must be the min, got {}", g.worst_case); + assert_eq!(g.sign_agreement, 2, "two instruments are net-positive"); + assert_eq!(g.per_instrument, vec![ + ("A".to_string(), 0.40), ("B".to_string(), -0.20), ("C".to_string(), 0.10), + ]); + } + + #[test] + fn generalization_is_not_lifted_by_a_strong_instrument() { + let a = report_with_r(0.0, 0.40, 0.0); + let b = report_with_r(0.0, -0.20, 0.0); + let strong = report_with_r(0.0, 99.0, 0.0); + let pairs = vec![("A".to_string(), &a), ("B".to_string(), &b), ("S".to_string(), &strong)]; + let g = generalization(&pairs, "expectancy_r").expect("R metric"); + assert!((g.worst_case - (-0.20)).abs() < 1e-12, "a strong instrument must not raise the floor"); + } + + #[test] + fn generalization_refuses_a_non_r_metric() { + let a = report_with_r(0.0, 0.4, 0.0); + let b = report_with_r(0.0, 0.2, 0.0); + let pairs = vec![("A".to_string(), &a), ("B".to_string(), &b)]; + match generalization(&pairs, "total_pips") { + Err(RegistryError::NonRMetric(m)) => assert_eq!(m, "total_pips"), + other => panic!("expected NonRMetric, got {other:?}"), + } + } + + #[test] + fn generalization_refuses_fewer_than_two_instruments() { + let a = report_with_r(0.0, 0.4, 0.0); + let pairs = vec![("A".to_string(), &a)]; + match generalization(&pairs, "expectancy_r") { + Err(RegistryError::TooFewInstruments(n)) => assert_eq!(n, 1), + other => panic!("expected TooFewInstruments(1), got {other:?}"), + } + } + + #[test] + fn generalization_reports_a_bad_metric_before_arity() { + // The doc-comment pins this ordering: the metric check precedes the + // arity check, so a bad metric is surfaced even on an empty slice + // (which would otherwise yield TooFewInstruments(0)). This decides + // which error the CLI shows when both are wrong; reversing the two + // guards in `generalization` would flip this to TooFewInstruments. + let pairs: Vec<(String, &RunReport)> = vec![]; + match generalization(&pairs, "total_pips") { + Err(RegistryError::NonRMetric(m)) => assert_eq!(m, "total_pips"), + other => panic!("expected NonRMetric before arity, got {other:?}"), + } + } + + #[test] + fn generalization_is_deterministic() { + let a = report_with_r(0.0, 0.40, 0.0); + let b = report_with_r(0.0, -0.20, 0.0); + let pairs = vec![("A".to_string(), &a), ("B".to_string(), &b)]; + let g1 = generalization(&pairs, "expectancy_r").expect("R metric"); + let g2 = generalization(&pairs, "expectancy_r").expect("R metric"); + assert_eq!(g1, g2, "a pure fold must be deterministic (C1)"); + } + + #[test] + fn check_r_metric_accepts_r_and_refuses_pip() { + assert!(check_r_metric("expectancy_r").is_ok()); + assert!(check_r_metric("sqn_normalized").is_ok()); + match check_r_metric("total_pips") { + Err(RegistryError::NonRMetric(m)) => assert_eq!(m, "total_pips"), + other => panic!("expected NonRMetric, got {other:?}"), + } + match check_r_metric("nope") { + Err(RegistryError::UnknownMetric(m)) => assert_eq!(m, "nope"), + other => panic!("expected UnknownMetric, got {other:?}"), + } + } } diff --git a/crates/aura-registry/src/lineage.rs b/crates/aura-registry/src/lineage.rs index 4ada357..00c0426 100644 --- a/crates/aura-registry/src/lineage.rs +++ b/crates/aura-registry/src/lineage.rs @@ -24,6 +24,7 @@ pub enum FamilyKind { Sweep, MonteCarlo, WalkForward, + CrossInstrument, } /// One persisted family member: a run record stamped with its lineage link. diff --git a/crates/aura-registry/src/trace_store.rs b/crates/aura-registry/src/trace_store.rs index c01b6c2..b93f868 100644 --- a/crates/aura-registry/src/trace_store.rs +++ b/crates/aura-registry/src/trace_store.rs @@ -254,6 +254,7 @@ mod tests { seed: 0, broker: "sim-optimal(pip_size=0.0001)".to_string(), selection: None, + instrument: None, } }