diff --git a/crates/aura-cli/src/research_docs.rs b/crates/aura-cli/src/research_docs.rs index 79b6216..700303c 100644 --- a/crates/aura-cli/src/research_docs.rs +++ b/crates/aura-cli/src/research_docs.rs @@ -97,6 +97,9 @@ fn doc_fault_prose(f: &DocFault) -> String { DocFault::GateAfterTerminalStage { stage } => { format!("stage {stage}: a gate cannot follow a terminal stage (monte_carlo/generalize)") } + DocFault::ZeroWalkForwardLength { stage, field } => { + format!("pipeline[{stage}]: walk_forward {field} must be > 0") + } DocFault::EmptyInstruments => "data.instruments is empty".into(), DocFault::NoWindow => "data.windows is empty".into(), DocFault::BadWindow { index } => { @@ -401,4 +404,11 @@ mod tests { ); } } + + #[test] + fn zero_walk_forward_length_prose_is_path_addressed_and_debug_free() { + let prose = doc_fault_prose(&DocFault::ZeroWalkForwardLength { stage: 2, field: "step_ms" }); + assert_eq!(prose, "pipeline[2]: walk_forward step_ms must be > 0"); + assert!(!prose.contains("ZeroWalkForwardLength"), "Debug leak: {prose}"); + } } diff --git a/crates/aura-cli/tests/research_docs.rs b/crates/aura-cli/tests/research_docs.rs index 656d86b..b9dff65 100644 --- a/crates/aura-cli/tests/research_docs.rs +++ b/crates/aura-cli/tests/research_docs.rs @@ -43,8 +43,8 @@ const PROCESS_DOC: &str = r#"{ { "block": "std::gate", "all": [ { "metric": "net_expectancy_r", "cmp": "gt", "value": 0.0 }, { "metric": "overfit_probability", "cmp": "lt", "value": 0.1 } ] }, - { "block": "std::walk_forward", "folds": 4, "in_sample_bars": 4000, - "out_of_sample_bars": 1000, "metric": "net_expectancy_r", "select": "argmax" } + { "block": "std::walk_forward", "in_sample_ms": 4000, "out_of_sample_ms": 1000, + "step_ms": 1000, "mode": "rolling", "metric": "net_expectancy_r", "select": "argmax" } ] }"#; @@ -78,6 +78,56 @@ fn process_validate_refuses_unknown_metric_as_prose_exit_1() { assert!(!out.contains("UnknownMetric"), "Debug leak: {out}"); } +#[test] +fn process_validate_refuses_zero_walk_forward_length_as_prose_exit_1() { + let dir = temp_cwd("process-validate-zero-wf"); + let bad = PROCESS_DOC.replacen("\"step_ms\": 1000", "\"step_ms\": 0", 1); + write_doc(&dir, "bad.process.json", &bad); + let (out, code) = run_code_in(&dir, &["process", "validate", "bad.process.json"]); + assert_eq!(code, Some(1), "stdout/stderr: {out}"); + assert!(out.contains("pipeline[2]: walk_forward step_ms must be > 0"), "stdout/stderr: {out}"); + assert!(!out.contains("ZeroWalkForwardLength"), "Debug leak: {out}"); +} + +/// A document authored against the vocabulary `std::walk_forward` carried +/// before cycle 0107 (`folds`/`in_sample_bars`/`out_of_sample_bars`, none of +/// which the engine's `WindowRoller` accepts) is refused end-to-end through +/// the CLI's malformed-document path (a parse failure, distinct from the +/// validate-fault path exercised above) with prose naming the offending +/// slot — never a Debug-formatted `DocError`, never a silent accept of the +/// retired shape. +#[test] +fn process_validate_refuses_retired_walk_forward_vocabulary_as_prose_exit_1() { + let dir = temp_cwd("process-validate-retired-wf-vocab"); + let bad = PROCESS_DOC.replacen( + "{ \"block\": \"std::walk_forward\", \"in_sample_ms\": 4000, \"out_of_sample_ms\": 1000,\n \"step_ms\": 1000, \"mode\": \"rolling\", \"metric\": \"net_expectancy_r\", \"select\": \"argmax\" }", + "{ \"block\": \"std::walk_forward\", \"folds\": 4, \"in_sample_bars\": 4000, \"out_of_sample_bars\": 1000, \"metric\": \"net_expectancy_r\", \"select\": \"argmax\" }", + 1, + ); + assert_ne!(bad, PROCESS_DOC, "replacen must actually match the fixture's walk_forward stage"); + write_doc(&dir, "bad.process.json", &bad); + let (out, code) = run_code_in(&dir, &["process", "validate", "bad.process.json"]); + assert_eq!(code, Some(1), "stdout/stderr: {out}"); + assert!(out.contains("unknown slot \"folds\""), "stdout/stderr: {out}"); + assert!(!out.contains("Malformed"), "Debug leak: {out}"); +} + +/// The roller's second `RollMode` — `anchored` (growing, origin-fixed +/// in-sample) — round-trips through the CLI's actual validate path, not +/// just the `slot_kind_label` description string: a data author who writes +/// `"mode": "anchored"` gets a valid document, exercising the `WfMode` +/// serde oracle's non-default variant end to end. +#[test] +fn process_validate_accepts_anchored_walk_forward_mode() { + let dir = temp_cwd("process-validate-anchored-wf-mode"); + let anchored = PROCESS_DOC.replacen("\"mode\": \"rolling\"", "\"mode\": \"anchored\"", 1); + assert_ne!(anchored, PROCESS_DOC, "replacen must actually match the fixture's mode field"); + write_doc(&dir, "anchored.process.json", &anchored); + let (out, code) = run_code_in(&dir, &["process", "validate", "anchored.process.json"]); + assert_eq!(code, Some(0), "stdout/stderr: {out}"); + assert!(out.contains("process document valid (intrinsic): 3 pipeline blocks, 2 gate predicates")); +} + #[test] fn process_introspect_vocabulary_block_and_content_id() { let (out, code) = run_code(&["process", "introspect", "--vocabulary"]); @@ -90,6 +140,14 @@ fn process_introspect_vocabulary_block_and_content_id() { assert!(out.contains("metric")); assert!(out.contains("required")); + let (out, code) = run_code(&["process", "introspect", "--block", "std::walk_forward"]); + assert_eq!(code, Some(0)); + for slot in ["in_sample_ms", "out_of_sample_ms", "step_ms", "mode"] { + assert!(out.contains(slot), "walk_forward describe misses {slot}: {out}"); + } + assert!(out.contains("one of: rolling | anchored"), "mode label missing: {out}"); + assert!(!out.contains("folds"), "retired slot still advertised: {out}"); + let dir = temp_cwd("process-introspect-content-id"); write_doc(&dir, "p.process.json", PROCESS_DOC); let (out, code) = run_code_in(&dir, &["process", "introspect", "--content-id", "p.process.json"]); diff --git a/crates/aura-research/src/lib.rs b/crates/aura-research/src/lib.rs index 1ea8c8e..c79dce3 100644 --- a/crates/aura-research/src/lib.rs +++ b/crates/aura-research/src/lib.rs @@ -56,9 +56,10 @@ pub enum StageBlock { Gate { all: Vec }, #[serde(rename = "std::walk_forward")] WalkForward { - folds: u32, - in_sample_bars: u64, - out_of_sample_bars: u64, + in_sample_ms: u64, + out_of_sample_ms: u64, + step_ms: u64, + mode: WfMode, metric: String, select: SelectRule, }, @@ -83,6 +84,17 @@ pub enum SelectRule { PlateauWorst, } +/// Walk-forward roll mode — wire form "rolling" | "anchored" (the two shipped +/// engine `RollMode`s: fixed-length rolling in-sample vs anchored-origin +/// growing in-sample). +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +pub enum WfMode { + #[serde(rename = "rolling")] + Rolling, + #[serde(rename = "anchored")] + Anchored, +} + /// One typed gate predicate: ` `. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -116,6 +128,8 @@ pub enum Cmp { pub enum SlotKind { MetricName, SelectRule, + /// Walk-forward roll mode (`WfMode` wire strings). + WfMode, Bool, U32, U64, @@ -164,9 +178,10 @@ pub const PROCESS_BLOCKS: &[BlockSchema] = &[ id: "std::walk_forward", doc: "rolling in-sample optimize + out-of-sample test", slots: &[ - SlotInfo { name: "folds", kind: SlotKind::U32, required: true }, - SlotInfo { name: "in_sample_bars", kind: SlotKind::U64, required: true }, - SlotInfo { name: "out_of_sample_bars", kind: SlotKind::U64, required: true }, + SlotInfo { name: "in_sample_ms", kind: SlotKind::U64, required: true }, + SlotInfo { name: "out_of_sample_ms", kind: SlotKind::U64, required: true }, + SlotInfo { name: "step_ms", kind: SlotKind::U64, required: true }, + SlotInfo { name: "mode", kind: SlotKind::WfMode, required: true }, SlotInfo { name: "metric", kind: SlotKind::MetricName, required: true }, SlotInfo { name: "select", kind: SlotKind::SelectRule, required: true }, ], @@ -248,6 +263,18 @@ fn select_from( .map_err(|_| format!("block {block}: unknown select rule \"{s}\"")) } +fn mode_from( + m: &serde_json::Map, + block: &str, +) -> Result { + let s = require_str(m, "mode", block)?; + // WfMode's own derived Deserialize (which carries the #[serde(rename)] + // wire strings) is the type oracle, mirroring select_from: no second + // hardcoded copy of the two roll-mode wire strings. + serde_json::from_value(serde_json::Value::String(s.clone())) + .map_err(|_| format!("block {block}: unknown mode \"{s}\"")) +} + fn stage_from_value(v: &serde_json::Value) -> Result { let m = v .as_object() @@ -280,9 +307,10 @@ fn stage_from_value(v: &serde_json::Value) -> Result { Ok(StageBlock::Gate { all }) } "std::walk_forward" => Ok(StageBlock::WalkForward { - folds: require_u32(m, "folds", &id)?, - in_sample_bars: require_u64(m, "in_sample_bars", &id)?, - out_of_sample_bars: require_u64(m, "out_of_sample_bars", &id)?, + in_sample_ms: require_u64(m, "in_sample_ms", &id)?, + out_of_sample_ms: require_u64(m, "out_of_sample_ms", &id)?, + step_ms: require_u64(m, "step_ms", &id)?, + mode: mode_from(m, &id)?, metric: require_str(m, "metric", &id)?, select: select_from(m, &id)?, }), @@ -564,6 +592,7 @@ pub enum DocFault { EmptyConjunction { stage: usize }, GateFirst, GateAfterTerminalStage { stage: usize }, + ZeroWalkForwardLength { stage: usize, field: &'static str }, // campaign side EmptyInstruments, NoWindow, @@ -589,13 +618,27 @@ pub fn validate_process(doc: &ProcessDoc) -> Vec { let mut terminal_seen = false; for (i, stage) in doc.pipeline.iter().enumerate() { match stage { - StageBlock::Sweep { metric, .. } - | StageBlock::Generalize { metric } - | StageBlock::WalkForward { metric, .. } => { + StageBlock::Sweep { metric, .. } | StageBlock::Generalize { metric } => { if !is_known_metric(metric) { faults.push(DocFault::UnknownMetric { stage: i, metric: metric.clone() }); } } + StageBlock::WalkForward { in_sample_ms, out_of_sample_ms, step_ms, metric, .. } => { + if !is_known_metric(metric) { + faults.push(DocFault::UnknownMetric { stage: i, metric: metric.clone() }); + } + // WindowRoller::new refuses NonPositiveLength at run time; the + // doc tier refuses the same zeroes earlier, path-addressed. + for (field, value) in [ + ("in_sample_ms", *in_sample_ms), + ("out_of_sample_ms", *out_of_sample_ms), + ("step_ms", *step_ms), + ] { + if value == 0 { + faults.push(DocFault::ZeroWalkForwardLength { stage: i, field }); + } + } + } StageBlock::Gate { all } => { if i == 0 { faults.push(DocFault::GateFirst); @@ -704,6 +747,7 @@ pub fn slot_kind_label(kind: SlotKind) -> &'static str { match kind { SlotKind::MetricName => "metric name (see metric_vocabulary)", SlotKind::SelectRule => "select rule: argmax | plateau:mean | plateau:worst", + SlotKind::WfMode => "one of: rolling | anchored", SlotKind::Bool => "bool", SlotKind::U32 => "non-negative integer (u32)", SlotKind::U64 => "non-negative integer (u64)", @@ -884,8 +928,8 @@ mod tests { { "block": "std::gate", "all": [ { "metric": "net_expectancy_r", "cmp": "gt", "value": 0.0 }, { "metric": "overfit_probability", "cmp": "lt", "value": 0.1 } ] }, - { "block": "std::walk_forward", "folds": 4, "in_sample_bars": 4000, - "out_of_sample_bars": 1000, "metric": "net_expectancy_r", "select": "argmax" } + { "block": "std::walk_forward", "in_sample_ms": 4000, "out_of_sample_ms": 1000, + "step_ms": 1000, "mode": "rolling", "metric": "net_expectancy_r", "select": "argmax" } ] }"#; @@ -911,6 +955,24 @@ mod tests { } other => panic!("stage 1 is not a gate: {other:?}"), } + match &doc.pipeline[2] { + StageBlock::WalkForward { + in_sample_ms, + out_of_sample_ms, + step_ms, + mode, + metric, + select, + } => { + assert_eq!(*in_sample_ms, 4000); + assert_eq!(*out_of_sample_ms, 1000); + assert_eq!(*step_ms, 1000); + assert_eq!(*mode, WfMode::Rolling); + assert_eq!(metric, "net_expectancy_r"); + assert_eq!(*select, SelectRule::Argmax); + } + other => panic!("stage 2 is not a walk_forward: {other:?}"), + } } #[test] @@ -1045,8 +1107,8 @@ mod tests { r#""pipeline":[{"block":"std::sweep","metric":"net_expectancy_r","select":"plateau:worst","deflate":true},"#, r#"{"block":"std::gate","all":[{"metric":"net_expectancy_r","cmp":"gt","value":0.0},"#, r#"{"metric":"overfit_probability","cmp":"lt","value":0.1}]},"#, - r#"{"block":"std::walk_forward","folds":4,"in_sample_bars":4000,"out_of_sample_bars":1000,"#, - r#""metric":"net_expectancy_r","select":"argmax"}]}"# + r#"{"block":"std::walk_forward","in_sample_ms":4000,"out_of_sample_ms":1000,"step_ms":1000,"#, + r#""mode":"rolling","metric":"net_expectancy_r","select":"argmax"}]}"# ); assert_eq!(canonical, golden); assert!(!canonical.ends_with('\n')); @@ -1140,6 +1202,76 @@ mod tests { .contains(&DocFault::GateAfterTerminalStage { stage: 2 })); } + #[test] + fn walk_forward_parses_both_modes_and_refuses_bad_slots() { + // "anchored" is the other accepted wire mode + let anchored = PROCESS_FIXTURE.replacen("\"mode\": \"rolling\"", "\"mode\": \"anchored\"", 1); + let doc = parse_process(&anchored).expect("anchored parses"); + assert!(matches!( + &doc.pipeline[2], + StageBlock::WalkForward { mode: WfMode::Anchored, .. } + )); + // an unknown mode string refuses through WfMode's serde oracle + let bad_mode = PROCESS_FIXTURE.replacen("\"mode\": \"rolling\"", "\"mode\": \"expanding\"", 1); + let err = parse_process(&bad_mode).expect_err("unknown mode refused"); + assert!(matches!(err, DocError::Malformed(msg) if msg.contains("unknown mode \"expanding\""))); + // the retired 0106 vocabulary ("folds") is an unknown slot now + let folds = PROCESS_FIXTURE.replacen("\"in_sample_ms\": 4000", "\"folds\": 4", 1); + let err = parse_process(&folds).expect_err("folds refused"); + assert!(matches!(err, DocError::Malformed(msg) if msg.contains("unknown slot \"folds\""))); + // a missing required length slot is named + let missing = PROCESS_FIXTURE.replacen("\"step_ms\": 1000, ", "", 1); + let err = parse_process(&missing).expect_err("missing step_ms refused"); + assert!( + matches!(err, DocError::Malformed(msg) if msg.contains("missing required slot \"step_ms\"")) + ); + } + + #[test] + fn validate_process_reports_each_zero_walk_forward_length() { + let ok = parse_process(PROCESS_FIXTURE).unwrap(); + let zeroed = ProcessDoc { + pipeline: vec![ + ok.pipeline[0].clone(), + StageBlock::WalkForward { + in_sample_ms: 0, + out_of_sample_ms: 0, + step_ms: 0, + mode: WfMode::Rolling, + metric: "net_expectancy_r".into(), + select: SelectRule::Argmax, + }, + ], + ..ok.clone() + }; + assert_eq!( + validate_process(&zeroed), + vec![ + DocFault::ZeroWalkForwardLength { stage: 1, field: "in_sample_ms" }, + DocFault::ZeroWalkForwardLength { stage: 1, field: "out_of_sample_ms" }, + DocFault::ZeroWalkForwardLength { stage: 1, field: "step_ms" }, + ] + ); + let one = ProcessDoc { + pipeline: vec![ + ok.pipeline[0].clone(), + StageBlock::WalkForward { + in_sample_ms: 4000, + out_of_sample_ms: 1000, + step_ms: 0, + mode: WfMode::Anchored, + metric: "net_expectancy_r".into(), + select: SelectRule::Argmax, + }, + ], + ..ok.clone() + }; + assert_eq!( + validate_process(&one), + vec![DocFault::ZeroWalkForwardLength { stage: 1, field: "step_ms" }] + ); + } + #[test] fn validate_campaign_accepts_the_fixture_and_reports_each_fault() { let ok = parse_campaign(CAMPAIGN_FIXTURE).unwrap(); @@ -1197,6 +1329,14 @@ mod tests { let sweep = describe_block("std::sweep").expect("sweep describable"); assert!(sweep.slots.iter().any(|s| s.name == "metric" && s.required)); assert!(sweep.slots.iter().any(|s| s.name == "deflate" && !s.required)); + let wf = describe_block("std::walk_forward").expect("walk_forward describable"); + let names: Vec<&str> = wf.slots.iter().map(|s| s.name).collect(); + assert_eq!( + names, + vec!["in_sample_ms", "out_of_sample_ms", "step_ms", "mode", "metric", "select"] + ); + let mode = wf.slots.iter().find(|s| s.name == "mode").expect("mode slot"); + assert_eq!(slot_kind_label(mode.kind), "one of: rolling | anchored"); assert!(describe_block("std::strategy").is_some()); assert!(describe_block("std::nope").is_none()); }