feat(research): std::walk_forward vocabulary corrected to machinery-true fields (0107 task 1)
folds/in_sample_bars/out_of_sample_bars mapped to nothing WindowRoller::new accepts; the block now carries in_sample_ms/ out_of_sample_ms/step_ms/mode (WfMode: rolling|anchored) — the roller's three lengths in the campaign window's own epoch-ms unit plus both shipped RollModes (#198 decision 2). New intrinsic fault ZeroWalkForwardLength (path-addressed) refuses zero lengths at the doc tier, earlier than the roller's NonPositiveLength. Golden canonical pin, PROCESS_FIXTURE, introspection tables, and the aura-cli prose/fixture twins moved together; fieldtest corpus untouched (historical record — stored walk_forward docs get new content ids by design). Gates: aura-research 14/14, aura-cli green, workspace build clean. refs #198
This commit is contained in:
@@ -97,6 +97,9 @@ fn doc_fault_prose(f: &DocFault) -> String {
|
||||
DocFault::GateAfterTerminalStage { stage } => {
|
||||
format!("stage {stage}: a gate cannot follow a terminal stage (monte_carlo/generalize)")
|
||||
}
|
||||
DocFault::ZeroWalkForwardLength { stage, field } => {
|
||||
format!("pipeline[{stage}]: walk_forward {field} must be > 0")
|
||||
}
|
||||
DocFault::EmptyInstruments => "data.instruments is empty".into(),
|
||||
DocFault::NoWindow => "data.windows is empty".into(),
|
||||
DocFault::BadWindow { index } => {
|
||||
@@ -401,4 +404,11 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_walk_forward_length_prose_is_path_addressed_and_debug_free() {
|
||||
let prose = doc_fault_prose(&DocFault::ZeroWalkForwardLength { stage: 2, field: "step_ms" });
|
||||
assert_eq!(prose, "pipeline[2]: walk_forward step_ms must be > 0");
|
||||
assert!(!prose.contains("ZeroWalkForwardLength"), "Debug leak: {prose}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,8 +43,8 @@ const PROCESS_DOC: &str = r#"{
|
||||
{ "block": "std::gate", "all": [
|
||||
{ "metric": "net_expectancy_r", "cmp": "gt", "value": 0.0 },
|
||||
{ "metric": "overfit_probability", "cmp": "lt", "value": 0.1 } ] },
|
||||
{ "block": "std::walk_forward", "folds": 4, "in_sample_bars": 4000,
|
||||
"out_of_sample_bars": 1000, "metric": "net_expectancy_r", "select": "argmax" }
|
||||
{ "block": "std::walk_forward", "in_sample_ms": 4000, "out_of_sample_ms": 1000,
|
||||
"step_ms": 1000, "mode": "rolling", "metric": "net_expectancy_r", "select": "argmax" }
|
||||
]
|
||||
}"#;
|
||||
|
||||
@@ -78,6 +78,56 @@ fn process_validate_refuses_unknown_metric_as_prose_exit_1() {
|
||||
assert!(!out.contains("UnknownMetric"), "Debug leak: {out}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn process_validate_refuses_zero_walk_forward_length_as_prose_exit_1() {
|
||||
let dir = temp_cwd("process-validate-zero-wf");
|
||||
let bad = PROCESS_DOC.replacen("\"step_ms\": 1000", "\"step_ms\": 0", 1);
|
||||
write_doc(&dir, "bad.process.json", &bad);
|
||||
let (out, code) = run_code_in(&dir, &["process", "validate", "bad.process.json"]);
|
||||
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
||||
assert!(out.contains("pipeline[2]: walk_forward step_ms must be > 0"), "stdout/stderr: {out}");
|
||||
assert!(!out.contains("ZeroWalkForwardLength"), "Debug leak: {out}");
|
||||
}
|
||||
|
||||
/// A document authored against the vocabulary `std::walk_forward` carried
|
||||
/// before cycle 0107 (`folds`/`in_sample_bars`/`out_of_sample_bars`, none of
|
||||
/// which the engine's `WindowRoller` accepts) is refused end-to-end through
|
||||
/// the CLI's malformed-document path (a parse failure, distinct from the
|
||||
/// validate-fault path exercised above) with prose naming the offending
|
||||
/// slot — never a Debug-formatted `DocError`, never a silent accept of the
|
||||
/// retired shape.
|
||||
#[test]
|
||||
fn process_validate_refuses_retired_walk_forward_vocabulary_as_prose_exit_1() {
|
||||
let dir = temp_cwd("process-validate-retired-wf-vocab");
|
||||
let bad = PROCESS_DOC.replacen(
|
||||
"{ \"block\": \"std::walk_forward\", \"in_sample_ms\": 4000, \"out_of_sample_ms\": 1000,\n \"step_ms\": 1000, \"mode\": \"rolling\", \"metric\": \"net_expectancy_r\", \"select\": \"argmax\" }",
|
||||
"{ \"block\": \"std::walk_forward\", \"folds\": 4, \"in_sample_bars\": 4000, \"out_of_sample_bars\": 1000, \"metric\": \"net_expectancy_r\", \"select\": \"argmax\" }",
|
||||
1,
|
||||
);
|
||||
assert_ne!(bad, PROCESS_DOC, "replacen must actually match the fixture's walk_forward stage");
|
||||
write_doc(&dir, "bad.process.json", &bad);
|
||||
let (out, code) = run_code_in(&dir, &["process", "validate", "bad.process.json"]);
|
||||
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
||||
assert!(out.contains("unknown slot \"folds\""), "stdout/stderr: {out}");
|
||||
assert!(!out.contains("Malformed"), "Debug leak: {out}");
|
||||
}
|
||||
|
||||
/// The roller's second `RollMode` — `anchored` (growing, origin-fixed
|
||||
/// in-sample) — round-trips through the CLI's actual validate path, not
|
||||
/// just the `slot_kind_label` description string: a data author who writes
|
||||
/// `"mode": "anchored"` gets a valid document, exercising the `WfMode`
|
||||
/// serde oracle's non-default variant end to end.
|
||||
#[test]
|
||||
fn process_validate_accepts_anchored_walk_forward_mode() {
|
||||
let dir = temp_cwd("process-validate-anchored-wf-mode");
|
||||
let anchored = PROCESS_DOC.replacen("\"mode\": \"rolling\"", "\"mode\": \"anchored\"", 1);
|
||||
assert_ne!(anchored, PROCESS_DOC, "replacen must actually match the fixture's mode field");
|
||||
write_doc(&dir, "anchored.process.json", &anchored);
|
||||
let (out, code) = run_code_in(&dir, &["process", "validate", "anchored.process.json"]);
|
||||
assert_eq!(code, Some(0), "stdout/stderr: {out}");
|
||||
assert!(out.contains("process document valid (intrinsic): 3 pipeline blocks, 2 gate predicates"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn process_introspect_vocabulary_block_and_content_id() {
|
||||
let (out, code) = run_code(&["process", "introspect", "--vocabulary"]);
|
||||
@@ -90,6 +140,14 @@ fn process_introspect_vocabulary_block_and_content_id() {
|
||||
assert!(out.contains("metric"));
|
||||
assert!(out.contains("required"));
|
||||
|
||||
let (out, code) = run_code(&["process", "introspect", "--block", "std::walk_forward"]);
|
||||
assert_eq!(code, Some(0));
|
||||
for slot in ["in_sample_ms", "out_of_sample_ms", "step_ms", "mode"] {
|
||||
assert!(out.contains(slot), "walk_forward describe misses {slot}: {out}");
|
||||
}
|
||||
assert!(out.contains("one of: rolling | anchored"), "mode label missing: {out}");
|
||||
assert!(!out.contains("folds"), "retired slot still advertised: {out}");
|
||||
|
||||
let dir = temp_cwd("process-introspect-content-id");
|
||||
write_doc(&dir, "p.process.json", PROCESS_DOC);
|
||||
let (out, code) = run_code_in(&dir, &["process", "introspect", "--content-id", "p.process.json"]);
|
||||
|
||||
+156
-16
@@ -56,9 +56,10 @@ pub enum StageBlock {
|
||||
Gate { all: Vec<Predicate> },
|
||||
#[serde(rename = "std::walk_forward")]
|
||||
WalkForward {
|
||||
folds: u32,
|
||||
in_sample_bars: u64,
|
||||
out_of_sample_bars: u64,
|
||||
in_sample_ms: u64,
|
||||
out_of_sample_ms: u64,
|
||||
step_ms: u64,
|
||||
mode: WfMode,
|
||||
metric: String,
|
||||
select: SelectRule,
|
||||
},
|
||||
@@ -83,6 +84,17 @@ pub enum SelectRule {
|
||||
PlateauWorst,
|
||||
}
|
||||
|
||||
/// Walk-forward roll mode — wire form "rolling" | "anchored" (the two shipped
|
||||
/// engine `RollMode`s: fixed-length rolling in-sample vs anchored-origin
|
||||
/// growing in-sample).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub enum WfMode {
|
||||
#[serde(rename = "rolling")]
|
||||
Rolling,
|
||||
#[serde(rename = "anchored")]
|
||||
Anchored,
|
||||
}
|
||||
|
||||
/// One typed gate predicate: `<metric> <cmp> <value>`.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -116,6 +128,8 @@ pub enum Cmp {
|
||||
pub enum SlotKind {
|
||||
MetricName,
|
||||
SelectRule,
|
||||
/// Walk-forward roll mode (`WfMode` wire strings).
|
||||
WfMode,
|
||||
Bool,
|
||||
U32,
|
||||
U64,
|
||||
@@ -164,9 +178,10 @@ pub const PROCESS_BLOCKS: &[BlockSchema] = &[
|
||||
id: "std::walk_forward",
|
||||
doc: "rolling in-sample optimize + out-of-sample test",
|
||||
slots: &[
|
||||
SlotInfo { name: "folds", kind: SlotKind::U32, required: true },
|
||||
SlotInfo { name: "in_sample_bars", kind: SlotKind::U64, required: true },
|
||||
SlotInfo { name: "out_of_sample_bars", kind: SlotKind::U64, required: true },
|
||||
SlotInfo { name: "in_sample_ms", kind: SlotKind::U64, required: true },
|
||||
SlotInfo { name: "out_of_sample_ms", kind: SlotKind::U64, required: true },
|
||||
SlotInfo { name: "step_ms", kind: SlotKind::U64, required: true },
|
||||
SlotInfo { name: "mode", kind: SlotKind::WfMode, required: true },
|
||||
SlotInfo { name: "metric", kind: SlotKind::MetricName, required: true },
|
||||
SlotInfo { name: "select", kind: SlotKind::SelectRule, required: true },
|
||||
],
|
||||
@@ -248,6 +263,18 @@ fn select_from(
|
||||
.map_err(|_| format!("block {block}: unknown select rule \"{s}\""))
|
||||
}
|
||||
|
||||
fn mode_from(
|
||||
m: &serde_json::Map<String, serde_json::Value>,
|
||||
block: &str,
|
||||
) -> Result<WfMode, String> {
|
||||
let s = require_str(m, "mode", block)?;
|
||||
// WfMode's own derived Deserialize (which carries the #[serde(rename)]
|
||||
// wire strings) is the type oracle, mirroring select_from: no second
|
||||
// hardcoded copy of the two roll-mode wire strings.
|
||||
serde_json::from_value(serde_json::Value::String(s.clone()))
|
||||
.map_err(|_| format!("block {block}: unknown mode \"{s}\""))
|
||||
}
|
||||
|
||||
fn stage_from_value(v: &serde_json::Value) -> Result<StageBlock, String> {
|
||||
let m = v
|
||||
.as_object()
|
||||
@@ -280,9 +307,10 @@ fn stage_from_value(v: &serde_json::Value) -> Result<StageBlock, String> {
|
||||
Ok(StageBlock::Gate { all })
|
||||
}
|
||||
"std::walk_forward" => Ok(StageBlock::WalkForward {
|
||||
folds: require_u32(m, "folds", &id)?,
|
||||
in_sample_bars: require_u64(m, "in_sample_bars", &id)?,
|
||||
out_of_sample_bars: require_u64(m, "out_of_sample_bars", &id)?,
|
||||
in_sample_ms: require_u64(m, "in_sample_ms", &id)?,
|
||||
out_of_sample_ms: require_u64(m, "out_of_sample_ms", &id)?,
|
||||
step_ms: require_u64(m, "step_ms", &id)?,
|
||||
mode: mode_from(m, &id)?,
|
||||
metric: require_str(m, "metric", &id)?,
|
||||
select: select_from(m, &id)?,
|
||||
}),
|
||||
@@ -564,6 +592,7 @@ pub enum DocFault {
|
||||
EmptyConjunction { stage: usize },
|
||||
GateFirst,
|
||||
GateAfterTerminalStage { stage: usize },
|
||||
ZeroWalkForwardLength { stage: usize, field: &'static str },
|
||||
// campaign side
|
||||
EmptyInstruments,
|
||||
NoWindow,
|
||||
@@ -589,13 +618,27 @@ pub fn validate_process(doc: &ProcessDoc) -> Vec<DocFault> {
|
||||
let mut terminal_seen = false;
|
||||
for (i, stage) in doc.pipeline.iter().enumerate() {
|
||||
match stage {
|
||||
StageBlock::Sweep { metric, .. }
|
||||
| StageBlock::Generalize { metric }
|
||||
| StageBlock::WalkForward { metric, .. } => {
|
||||
StageBlock::Sweep { metric, .. } | StageBlock::Generalize { metric } => {
|
||||
if !is_known_metric(metric) {
|
||||
faults.push(DocFault::UnknownMetric { stage: i, metric: metric.clone() });
|
||||
}
|
||||
}
|
||||
StageBlock::WalkForward { in_sample_ms, out_of_sample_ms, step_ms, metric, .. } => {
|
||||
if !is_known_metric(metric) {
|
||||
faults.push(DocFault::UnknownMetric { stage: i, metric: metric.clone() });
|
||||
}
|
||||
// WindowRoller::new refuses NonPositiveLength at run time; the
|
||||
// doc tier refuses the same zeroes earlier, path-addressed.
|
||||
for (field, value) in [
|
||||
("in_sample_ms", *in_sample_ms),
|
||||
("out_of_sample_ms", *out_of_sample_ms),
|
||||
("step_ms", *step_ms),
|
||||
] {
|
||||
if value == 0 {
|
||||
faults.push(DocFault::ZeroWalkForwardLength { stage: i, field });
|
||||
}
|
||||
}
|
||||
}
|
||||
StageBlock::Gate { all } => {
|
||||
if i == 0 {
|
||||
faults.push(DocFault::GateFirst);
|
||||
@@ -704,6 +747,7 @@ pub fn slot_kind_label(kind: SlotKind) -> &'static str {
|
||||
match kind {
|
||||
SlotKind::MetricName => "metric name (see metric_vocabulary)",
|
||||
SlotKind::SelectRule => "select rule: argmax | plateau:mean | plateau:worst",
|
||||
SlotKind::WfMode => "one of: rolling | anchored",
|
||||
SlotKind::Bool => "bool",
|
||||
SlotKind::U32 => "non-negative integer (u32)",
|
||||
SlotKind::U64 => "non-negative integer (u64)",
|
||||
@@ -884,8 +928,8 @@ mod tests {
|
||||
{ "block": "std::gate", "all": [
|
||||
{ "metric": "net_expectancy_r", "cmp": "gt", "value": 0.0 },
|
||||
{ "metric": "overfit_probability", "cmp": "lt", "value": 0.1 } ] },
|
||||
{ "block": "std::walk_forward", "folds": 4, "in_sample_bars": 4000,
|
||||
"out_of_sample_bars": 1000, "metric": "net_expectancy_r", "select": "argmax" }
|
||||
{ "block": "std::walk_forward", "in_sample_ms": 4000, "out_of_sample_ms": 1000,
|
||||
"step_ms": 1000, "mode": "rolling", "metric": "net_expectancy_r", "select": "argmax" }
|
||||
]
|
||||
}"#;
|
||||
|
||||
@@ -911,6 +955,24 @@ mod tests {
|
||||
}
|
||||
other => panic!("stage 1 is not a gate: {other:?}"),
|
||||
}
|
||||
match &doc.pipeline[2] {
|
||||
StageBlock::WalkForward {
|
||||
in_sample_ms,
|
||||
out_of_sample_ms,
|
||||
step_ms,
|
||||
mode,
|
||||
metric,
|
||||
select,
|
||||
} => {
|
||||
assert_eq!(*in_sample_ms, 4000);
|
||||
assert_eq!(*out_of_sample_ms, 1000);
|
||||
assert_eq!(*step_ms, 1000);
|
||||
assert_eq!(*mode, WfMode::Rolling);
|
||||
assert_eq!(metric, "net_expectancy_r");
|
||||
assert_eq!(*select, SelectRule::Argmax);
|
||||
}
|
||||
other => panic!("stage 2 is not a walk_forward: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1045,8 +1107,8 @@ mod tests {
|
||||
r#""pipeline":[{"block":"std::sweep","metric":"net_expectancy_r","select":"plateau:worst","deflate":true},"#,
|
||||
r#"{"block":"std::gate","all":[{"metric":"net_expectancy_r","cmp":"gt","value":0.0},"#,
|
||||
r#"{"metric":"overfit_probability","cmp":"lt","value":0.1}]},"#,
|
||||
r#"{"block":"std::walk_forward","folds":4,"in_sample_bars":4000,"out_of_sample_bars":1000,"#,
|
||||
r#""metric":"net_expectancy_r","select":"argmax"}]}"#
|
||||
r#"{"block":"std::walk_forward","in_sample_ms":4000,"out_of_sample_ms":1000,"step_ms":1000,"#,
|
||||
r#""mode":"rolling","metric":"net_expectancy_r","select":"argmax"}]}"#
|
||||
);
|
||||
assert_eq!(canonical, golden);
|
||||
assert!(!canonical.ends_with('\n'));
|
||||
@@ -1140,6 +1202,76 @@ mod tests {
|
||||
.contains(&DocFault::GateAfterTerminalStage { stage: 2 }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn walk_forward_parses_both_modes_and_refuses_bad_slots() {
|
||||
// "anchored" is the other accepted wire mode
|
||||
let anchored = PROCESS_FIXTURE.replacen("\"mode\": \"rolling\"", "\"mode\": \"anchored\"", 1);
|
||||
let doc = parse_process(&anchored).expect("anchored parses");
|
||||
assert!(matches!(
|
||||
&doc.pipeline[2],
|
||||
StageBlock::WalkForward { mode: WfMode::Anchored, .. }
|
||||
));
|
||||
// an unknown mode string refuses through WfMode's serde oracle
|
||||
let bad_mode = PROCESS_FIXTURE.replacen("\"mode\": \"rolling\"", "\"mode\": \"expanding\"", 1);
|
||||
let err = parse_process(&bad_mode).expect_err("unknown mode refused");
|
||||
assert!(matches!(err, DocError::Malformed(msg) if msg.contains("unknown mode \"expanding\"")));
|
||||
// the retired 0106 vocabulary ("folds") is an unknown slot now
|
||||
let folds = PROCESS_FIXTURE.replacen("\"in_sample_ms\": 4000", "\"folds\": 4", 1);
|
||||
let err = parse_process(&folds).expect_err("folds refused");
|
||||
assert!(matches!(err, DocError::Malformed(msg) if msg.contains("unknown slot \"folds\"")));
|
||||
// a missing required length slot is named
|
||||
let missing = PROCESS_FIXTURE.replacen("\"step_ms\": 1000, ", "", 1);
|
||||
let err = parse_process(&missing).expect_err("missing step_ms refused");
|
||||
assert!(
|
||||
matches!(err, DocError::Malformed(msg) if msg.contains("missing required slot \"step_ms\""))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_process_reports_each_zero_walk_forward_length() {
|
||||
let ok = parse_process(PROCESS_FIXTURE).unwrap();
|
||||
let zeroed = ProcessDoc {
|
||||
pipeline: vec![
|
||||
ok.pipeline[0].clone(),
|
||||
StageBlock::WalkForward {
|
||||
in_sample_ms: 0,
|
||||
out_of_sample_ms: 0,
|
||||
step_ms: 0,
|
||||
mode: WfMode::Rolling,
|
||||
metric: "net_expectancy_r".into(),
|
||||
select: SelectRule::Argmax,
|
||||
},
|
||||
],
|
||||
..ok.clone()
|
||||
};
|
||||
assert_eq!(
|
||||
validate_process(&zeroed),
|
||||
vec![
|
||||
DocFault::ZeroWalkForwardLength { stage: 1, field: "in_sample_ms" },
|
||||
DocFault::ZeroWalkForwardLength { stage: 1, field: "out_of_sample_ms" },
|
||||
DocFault::ZeroWalkForwardLength { stage: 1, field: "step_ms" },
|
||||
]
|
||||
);
|
||||
let one = ProcessDoc {
|
||||
pipeline: vec![
|
||||
ok.pipeline[0].clone(),
|
||||
StageBlock::WalkForward {
|
||||
in_sample_ms: 4000,
|
||||
out_of_sample_ms: 1000,
|
||||
step_ms: 0,
|
||||
mode: WfMode::Anchored,
|
||||
metric: "net_expectancy_r".into(),
|
||||
select: SelectRule::Argmax,
|
||||
},
|
||||
],
|
||||
..ok.clone()
|
||||
};
|
||||
assert_eq!(
|
||||
validate_process(&one),
|
||||
vec![DocFault::ZeroWalkForwardLength { stage: 1, field: "step_ms" }]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_campaign_accepts_the_fixture_and_reports_each_fault() {
|
||||
let ok = parse_campaign(CAMPAIGN_FIXTURE).unwrap();
|
||||
@@ -1197,6 +1329,14 @@ mod tests {
|
||||
let sweep = describe_block("std::sweep").expect("sweep describable");
|
||||
assert!(sweep.slots.iter().any(|s| s.name == "metric" && s.required));
|
||||
assert!(sweep.slots.iter().any(|s| s.name == "deflate" && !s.required));
|
||||
let wf = describe_block("std::walk_forward").expect("walk_forward describable");
|
||||
let names: Vec<&str> = wf.slots.iter().map(|s| s.name).collect();
|
||||
assert_eq!(
|
||||
names,
|
||||
vec!["in_sample_ms", "out_of_sample_ms", "step_ms", "mode", "metric", "select"]
|
||||
);
|
||||
let mode = wf.slots.iter().find(|s| s.name == "mode").expect("mode slot");
|
||||
assert_eq!(slot_kind_label(mode.kind), "one of: rolling | anchored");
|
||||
assert!(describe_block("std::strategy").is_some());
|
||||
assert!(describe_block("std::nope").is_none());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user