feat(0078): cross-instrument generalization as a validation step
Last piece of the Inferential-validation milestone (#146): a new `aura generalize` subcommand grades how consistently a single candidate holds across a set of instruments — the across-instrument axis of the same anti-false-discovery concern as trials-deflation (#144, within one family) and plateau-over-peak (#145, within one instrument's surface). Aggregator/validator, not a third selector (user-ratified): it runs a brought candidate (a single-cell stage1-r grid — --fast/--slow/--stop- length/--stop-k pinned to single values) across an instrument list (--real SYM1,SYM2,...), collects per-instrument R-metrics, and reduces them to a worst-case floor (min over instruments) + a sign-agreement count + the per-instrument breakdown. R-only (C10 — R is the only account/instrument-agnostic unit); the worst-case min is the cross- instrument sibling of PlateauMode::Worst and satisfies the non- domination constraint by construction (a strong instrument cannot lift a min). Rejected: a cross-instrument t-stat mean/std*sqrt(M), which reintroduces the pooled mean the milestone warns against. - RunManifest gains a first-class `instrument` lineage field (serde- widened, C14/C18 — legacy lines load as None, existing manifest bytes unchanged); the compat read-mirror threads it in lockstep. - FamilyKind::CrossInstrument (C12 comparison axis); the M per-instrument runs persist as a CrossInstrument family, each member self-identifying via its stamped instrument. The aggregate is recomputable from the members (McAggregate precedent), printed live. - The generalization reduction lives registry-side beside optimize_ deflated/optimize_plateau (C9), reusing resolve_metric/metric_value/ is_r_metric; check_r_metric is the data-free pre-check the CLI runs before evaluating any instrument. - Additive: existing sweep/walkforward/mc/standalone paths stay byte- identical (C23 — instrument omitted when None via skip_serializing_if). Verified by the orchestrator: cargo test --workspace green (0 failed — the new generalization_*, check_r_metric, parse_generalize_*, and the gated cross-instrument E2E all pass; the E2E ran the real GER40/USDJPY path on this host, exit 0); cargo clippy --workspace --all-targets -D warnings clean. closes #146
This commit is contained in:
+195
-4
@@ -24,9 +24,9 @@ use aura_engine::{
|
||||
VecSource, WalkForwardResult, WindowBounds, WindowRoller, WindowRun,
|
||||
};
|
||||
use aura_registry::{
|
||||
group_families, mc_member_reports, optimize_deflated, optimize_plateau, rank_by,
|
||||
sweep_member_reports, walkforward_member_reports, FamilyKind, FamilyMember, NameKind,
|
||||
PlateauMode, Registry, RunTraces, TraceStore, WriteKind,
|
||||
check_r_metric, generalization, group_families, mc_member_reports, optimize_deflated,
|
||||
optimize_plateau, rank_by, sweep_member_reports, walkforward_member_reports, FamilyKind,
|
||||
FamilyMember, Generalization, NameKind, PlateauMode, Registry, RunTraces, TraceStore, WriteKind,
|
||||
};
|
||||
use aura_std::{
|
||||
Add, Bias, Delay, Ema, GatedRecorder, Gt, Latch, LinComb, LongOnly, Mul, Recorder, RollingMax,
|
||||
@@ -174,6 +174,7 @@ fn sim_optimal_manifest(
|
||||
seed,
|
||||
broker: format!("sim-optimal(pip_size={pip_size})"),
|
||||
selection: None,
|
||||
instrument: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1662,6 +1663,69 @@ fn parse_sweep_args(
|
||||
Ok((strategy, name, persist, real.finish(&usage)?, grid))
|
||||
}
|
||||
|
||||
/// Parse the `generalize` tail:
|
||||
/// `[--strategy stage1-r] --real <SYM1,SYM2,...> --fast <n> --slow <n> --stop-length <n>
|
||||
/// --stop-k <f> [--from <ms>] [--to <ms>] [--metric <r-metric>] [--name <n>]`.
|
||||
/// The candidate is a single cell: each grid knob takes exactly one value and all
|
||||
/// four are required. `--real` is a comma list of >= 2 distinct non-empty symbols.
|
||||
/// Pure (no I/O / exit) so the grammar is unit-testable.
|
||||
#[allow(clippy::type_complexity)]
|
||||
fn parse_generalize_args(
|
||||
rest: &[&str],
|
||||
) -> Result<(String, Vec<String>, Stage1RGrid, String, Option<i64>, Option<i64>), String> {
|
||||
let usage = || "generalize [--strategy stage1-r] --real <SYM1,SYM2,...> --fast <n> --slow <n> --stop-length <n> --stop-k <f> [--from <ms>] [--to <ms>] [--metric <r-metric>] [--name <n>]".to_string();
|
||||
let mut symbols: Option<Vec<String>> = None;
|
||||
let mut from_ms: Option<i64> = None;
|
||||
let mut to_ms: Option<i64> = None;
|
||||
let mut fast: Option<i64> = None;
|
||||
let mut slow: Option<i64> = None;
|
||||
let mut stop_length: Option<i64> = None;
|
||||
let mut stop_k: Option<f64> = None;
|
||||
let mut metric = "expectancy_r".to_string();
|
||||
let mut name = "generalize".to_string();
|
||||
// a single grid knob, required and single-valued (csv with exactly one item)
|
||||
let one = |value: &str, usage: &dyn Fn() -> String| -> Result<(), String> {
|
||||
let items: Vec<&str> = value.split(',').collect();
|
||||
if items.len() != 1 || items[0].is_empty() { return Err(usage()); }
|
||||
Ok(())
|
||||
};
|
||||
let mut tail = rest;
|
||||
while let Some((flag, t)) = tail.split_first() {
|
||||
let (value, t) = t.split_first().ok_or_else(usage)?;
|
||||
match *flag {
|
||||
"--strategy" => { if *value != "stage1-r" { return Err(usage()); } }
|
||||
"--real" => {
|
||||
let parts: Vec<String> = value.split(',').map(|s| s.to_string()).collect();
|
||||
if parts.iter().any(|s| s.is_empty()) { return Err(usage()); }
|
||||
symbols = Some(parts);
|
||||
}
|
||||
"--from" => from_ms = Some(value.parse().map_err(|_| usage())?),
|
||||
"--to" => to_ms = Some(value.parse().map_err(|_| usage())?),
|
||||
"--fast" => { one(value, &usage)?; fast = Some(value.parse().map_err(|_| usage())?); }
|
||||
"--slow" => { one(value, &usage)?; slow = Some(value.parse().map_err(|_| usage())?); }
|
||||
"--stop-length" => { one(value, &usage)?; stop_length = Some(value.parse().map_err(|_| usage())?); }
|
||||
"--stop-k" => { one(value, &usage)?; stop_k = Some(value.parse().map_err(|_| usage())?); }
|
||||
"--metric" => metric = (*value).to_string(),
|
||||
"--name" => name = (*value).to_string(),
|
||||
_ => return Err(usage()),
|
||||
}
|
||||
tail = t;
|
||||
}
|
||||
let symbols = symbols.ok_or_else(usage)?;
|
||||
if symbols.len() < 2 { return Err(usage()); }
|
||||
// distinct: a duplicate would double-count one instrument in the floor / sign count
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
if !symbols.iter().all(|s| seen.insert(s.clone())) { return Err(usage()); }
|
||||
let grid = Stage1RGrid {
|
||||
fast: vec![fast.ok_or_else(usage)?],
|
||||
slow: vec![slow.ok_or_else(usage)?],
|
||||
stop_length: vec![stop_length.ok_or_else(usage)?],
|
||||
stop_k: vec![stop_k.ok_or_else(usage)?],
|
||||
..Stage1RGrid::default()
|
||||
};
|
||||
Ok((name, symbols, grid, metric, from_ms, to_ms))
|
||||
}
|
||||
|
||||
/// Parse the `walkforward` tail:
|
||||
/// `[--strategy <sma|momentum|stage1-r|stage1-breakout|stage1-meanrev>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--name <n> | --trace <n>] [--fast <csv>] [--slow <csv>] [--stop-length <csv>] [--stop-k <csv>] [--select <argmax|plateau:mean|plateau:worst>]`.
|
||||
/// Defaults: SMA-cross, synthetic, name "walkforward", no persist. `--name`/`--trace`
|
||||
@@ -1767,6 +1831,48 @@ fn run_sweep(strategy: Strategy, name: &str, persist: bool, data: DataSource, gr
|
||||
}
|
||||
}
|
||||
|
||||
/// `aura generalize --real <SYM1,SYM2,...> --fast <n> --slow <n> --stop-length <n>
|
||||
/// --stop-k <f>`: grade one stage1-r candidate (a single-cell grid) across an
|
||||
/// instrument list. Pre-checks the R-metric data-free (refuse a non-R / unknown
|
||||
/// metric before any run), runs the candidate per instrument (stamping each report's
|
||||
/// manifest `instrument`), reduces to the worst-case floor + sign-agreement +
|
||||
/// per-instrument breakdown (`generalization`, C9/C10), prints the aggregate, and
|
||||
/// persists the per-instrument members as a `CrossInstrument` family (C12/C18).
|
||||
fn run_generalize(
|
||||
name: &str,
|
||||
symbols: &[String],
|
||||
grid: &Stage1RGrid,
|
||||
metric: &str,
|
||||
from_ms: Option<i64>,
|
||||
to_ms: Option<i64>,
|
||||
) {
|
||||
// data-free metric pre-check: refuse a non-R / unknown metric before any run.
|
||||
if let Err(e) = check_r_metric(metric) {
|
||||
eprintln!("aura: {e}");
|
||||
std::process::exit(2);
|
||||
}
|
||||
let mut members: Vec<RunReport> = Vec::new();
|
||||
for symbol in symbols {
|
||||
let choice = DataChoice::Real { symbol: symbol.clone(), from_ms, to_ms };
|
||||
let data = DataSource::from_choice(choice); // per-instrument pip_or_refuse + has_symbol
|
||||
let family = stage1_r_sweep_family(None, &data, grid); // single-cell grid -> 1 member
|
||||
let mut report = family.points[0].report.clone();
|
||||
report.manifest.instrument = Some(symbol.clone());
|
||||
members.push(report);
|
||||
}
|
||||
let pairs: Vec<(String, &RunReport)> = symbols.iter().cloned().zip(members.iter()).collect();
|
||||
let agg = match generalization(&pairs, metric) {
|
||||
Ok(a) => a,
|
||||
Err(e) => { eprintln!("aura: {e}"); std::process::exit(2); }
|
||||
};
|
||||
println!("{}", generalize_json(&agg));
|
||||
let reg = default_registry();
|
||||
match reg.append_family(name, FamilyKind::CrossInstrument, &members) {
|
||||
Ok(id) => println!("{{\"family_id\":\"{id}\"}}"),
|
||||
Err(e) => { eprintln!("aura: failed to persist family: {e}"); std::process::exit(1); }
|
||||
}
|
||||
}
|
||||
|
||||
/// `aura walkforward [--name <n>|--trace <n>]`: run a built-in rolling walk-forward
|
||||
/// over the sample blueprint + a synthetic windowed source. Per window: sweep the
|
||||
/// built-in grid on the in-sample slice, optimize by total_pips (axis 2 inside axis 3,
|
||||
@@ -1983,6 +2089,25 @@ fn walkforward_summary_json(result: &WalkForwardResult) -> String {
|
||||
serde_json::json!({ "walkforward": obj }).to_string()
|
||||
}
|
||||
|
||||
/// The cross-instrument generalization line: the chosen metric, instrument count,
|
||||
/// worst-case floor, sign-agreement count, and the per-instrument breakdown. Canonical
|
||||
/// JSON (C14), mirroring `walkforward_summary_json`'s `{"generalize": obj}` shape.
|
||||
fn generalize_json(agg: &Generalization) -> String {
|
||||
let per: Vec<serde_json::Value> = agg
|
||||
.per_instrument
|
||||
.iter()
|
||||
.map(|(sym, v)| serde_json::json!([sym, v]))
|
||||
.collect();
|
||||
let obj = serde_json::json!({
|
||||
"metric": agg.selection_metric,
|
||||
"n_instruments": agg.n_instruments,
|
||||
"worst_case": agg.worst_case,
|
||||
"sign_agreement": agg.sign_agreement,
|
||||
"per_instrument": per,
|
||||
});
|
||||
serde_json::json!({ "generalize": obj }).to_string()
|
||||
}
|
||||
|
||||
/// A longer deterministic stream than `showcase_prices` — enough for several
|
||||
/// IS/OOS windows with SMA warm-up. Seed-determined via `SyntheticSpec` (C1).
|
||||
fn walkforward_prices() -> Vec<(Timestamp, Scalar)> {
|
||||
@@ -2954,7 +3079,7 @@ fn run_dispatch(args: RunArgs) -> Result<RunReport, String> {
|
||||
}
|
||||
|
||||
const USAGE: &str =
|
||||
"usage: aura run [--harness <sma|macd|stage1-r>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--trace <name>] | aura chart <name> [--tap <t>] [--panels] | aura graph | aura sweep [--strategy <sma|momentum|stage1-r|stage1-breakout|stage1-meanrev>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--name <n>|--trace <n>] | aura mc [--name <n>|--trace <n>] | aura mc --strategy stage1-r [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--fast <csv>] [--slow <csv>] [--stop-length <csv>] [--stop-k <csv>] [--block-len <n>] [--resamples <n>] [--seed <n>] | aura walkforward [--strategy <sma|momentum|stage1-r|stage1-breakout|stage1-meanrev>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--name <n>|--trace <n>] [--fast <csv>] [--slow <csv>] [--stop-length <csv>] [--stop-k <csv>] | aura runs families | aura runs family <id> [rank <metric>]";
|
||||
"usage: aura run [--harness <sma|macd|stage1-r>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--trace <name>] | aura chart <name> [--tap <t>] [--panels] | aura graph | aura sweep [--strategy <sma|momentum|stage1-r|stage1-breakout|stage1-meanrev>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--name <n>|--trace <n>] | aura mc [--name <n>|--trace <n>] | aura mc --strategy stage1-r [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--fast <csv>] [--slow <csv>] [--stop-length <csv>] [--stop-k <csv>] [--block-len <n>] [--resamples <n>] [--seed <n>] | aura walkforward [--strategy <sma|momentum|stage1-r|stage1-breakout|stage1-meanrev>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--name <n>|--trace <n>] [--fast <csv>] [--slow <csv>] [--stop-length <csv>] [--stop-k <csv>] | aura generalize [--strategy stage1-r] --real <SYM1,SYM2,...> --fast <n> --slow <n> --stop-length <n> --stop-k <f> [--from <ms>] [--to <ms>] [--metric <r-metric>] [--name <n>] | aura runs families | aura runs family <id> [rank <metric>]";
|
||||
|
||||
fn main() {
|
||||
// Collect argv and match the whole vector: every accepted form is exhaustive,
|
||||
@@ -3008,6 +3133,15 @@ fn main() {
|
||||
std::process::exit(2);
|
||||
}
|
||||
},
|
||||
["generalize", rest @ ..] => match parse_generalize_args(rest) {
|
||||
Ok((name, symbols, grid, metric, from_ms, to_ms)) => {
|
||||
run_generalize(&name, &symbols, &grid, &metric, from_ms, to_ms)
|
||||
}
|
||||
Err(msg) => {
|
||||
eprintln!("aura: {msg}");
|
||||
std::process::exit(2);
|
||||
}
|
||||
},
|
||||
["mc", rest @ ..] => match parse_mc_args(rest) {
|
||||
Ok(McArgs::Synthetic { name, persist }) => run_mc(&name, persist),
|
||||
Ok(McArgs::RealR { choice, grid, block_len, n_resamples, seed }) => {
|
||||
@@ -3952,6 +4086,63 @@ mod tests {
|
||||
assert!(parse_sweep_args(&["--channel", ""]).is_err()); // empty item
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_generalize_requires_a_single_value_candidate() {
|
||||
// a candidate is one cell: a multi-value grid flag is refused
|
||||
assert!(parse_generalize_args(&[
|
||||
"--real", "GER40,USDJPY", "--fast", "2,3", "--slow", "12",
|
||||
"--stop-length", "14", "--stop-k", "2.0",
|
||||
]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_generalize_requires_all_four_knobs() {
|
||||
// --slow absent -> refuse (no defaulting to the multi-value sweep grid)
|
||||
assert!(parse_generalize_args(&[
|
||||
"--real", "GER40,USDJPY", "--fast", "3",
|
||||
"--stop-length", "14", "--stop-k", "2.0",
|
||||
]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_generalize_refuses_fewer_than_two_instruments() {
|
||||
assert!(parse_generalize_args(&[
|
||||
"--real", "GER40", "--fast", "3", "--slow", "12",
|
||||
"--stop-length", "14", "--stop-k", "2.0",
|
||||
]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_generalize_refuses_a_duplicate_instrument() {
|
||||
assert!(parse_generalize_args(&[
|
||||
"--real", "GER40,GER40", "--fast", "3", "--slow", "12",
|
||||
"--stop-length", "14", "--stop-k", "2.0",
|
||||
]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_generalize_refuses_a_non_stage1r_strategy() {
|
||||
assert!(parse_generalize_args(&[
|
||||
"--strategy", "sma", "--real", "GER40,USDJPY", "--fast", "3", "--slow", "12",
|
||||
"--stop-length", "14", "--stop-k", "2.0",
|
||||
]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_generalize_defaults_name_and_metric() {
|
||||
let (name, symbols, grid, metric, _from, _to) = parse_generalize_args(&[
|
||||
"--real", "GER40,USDJPY", "--fast", "3", "--slow", "12",
|
||||
"--stop-length", "14", "--stop-k", "2.0",
|
||||
]).expect("valid generalize args");
|
||||
assert_eq!(name, "generalize");
|
||||
assert_eq!(metric, "expectancy_r");
|
||||
assert_eq!(symbols, vec!["GER40".to_string(), "USDJPY".to_string()]);
|
||||
assert_eq!(grid.fast, vec![3]);
|
||||
assert_eq!(grid.slow, vec![12]);
|
||||
assert_eq!(grid.stop_length, vec![14]);
|
||||
assert_eq!(grid.stop_k, vec![2.0]);
|
||||
}
|
||||
|
||||
/// Property: the meanrev `--window` / `--band-k` flags parse comma-separated
|
||||
/// lists onto `Stage1RGrid.{window,band_k}` (the meanrev family's signal axes),
|
||||
/// with the same strictness as the other grid flags — absent flags keep the
|
||||
|
||||
Reference in New Issue
Block a user