feat(campaign,registry,engine): record widenings + preflight v2 fault set (0108 tasks 1-2)
Task 1: RBootstrap and Generalization gain serde derives; lineage.rs gains StageBootstrap (per_survivor | pooled_oos) and CampaignGeneralization; StageRealization.bootstrap and CampaignRunRecord.generalizations land serde-default sparse (a pre-0108 campaign_runs.jsonl line still parses — pinned), threaded None/empty through every literal site. No behaviour change. Task 2: ExecFault::UnsupportedStage is gone; the preflight now admits the v2 shape sweep [gate]* [wf]? [mc]? [generalize]? via a monotone rank walk (precise per-pair PipelineShape prose) and statically refuses: single-instrument generalize (GeneralizeNeedsInstruments), non-R generalize metrics via check_r_metric (GeneralizeNonRMetric — no fourth roster site), zero mc params (ZeroBootstrapParam). CLI prose swapped accordingly. The loop's E2E phase added a tier-boundary pin: process validate accepts [sweep, mc, walk_forward] while campaign run refuses it data-free. KNOWN RED (documented in the plan, flips in task 5): crates/aura-cli/tests/research_docs.rs::campaign_run_v1_boundary_refuses_mc_process still pins the retired 'not executable in v1' prose. Interim gates: registry 57/0, campaign 33/0, cli --bin 111/0, workspace build clean. refs #200
This commit is contained in:
@@ -145,6 +145,7 @@ pub fn execute(
|
||||
run: 0,
|
||||
seed: campaign.seed,
|
||||
cells: cells_rec,
|
||||
generalizations: Vec::new(),
|
||||
};
|
||||
let run = registry.append_campaign_run(&record).map_err(ExecFault::Registry)?;
|
||||
record.run = run;
|
||||
@@ -204,6 +205,7 @@ fn run_cell(
|
||||
family_id: Some(family_id.clone()),
|
||||
survivor_ordinals: None,
|
||||
selection: Some(StageSelection { winner_ordinal, params, selection }),
|
||||
bootstrap: None,
|
||||
});
|
||||
// The whole family flows on (decision 3: `select` names the
|
||||
// recorded selection, never a filter).
|
||||
@@ -240,6 +242,7 @@ fn run_cell(
|
||||
family_id: None,
|
||||
survivor_ordinals: Some(ordinals),
|
||||
selection: None,
|
||||
bootstrap: None,
|
||||
});
|
||||
if empty {
|
||||
break; // decision 8: truncate this cell, keep running cells
|
||||
@@ -273,6 +276,7 @@ fn run_cell(
|
||||
family_id: Some(fam.family_id.clone()),
|
||||
survivor_ordinals: None,
|
||||
selection: None,
|
||||
bootstrap: None,
|
||||
});
|
||||
families.push(fam);
|
||||
}
|
||||
|
||||
+260
-50
@@ -18,7 +18,7 @@ use std::collections::BTreeMap;
|
||||
|
||||
use aura_core::Scalar;
|
||||
use aura_engine::RunReport;
|
||||
use aura_registry::RegistryError;
|
||||
use aura_registry::{check_r_metric, RegistryError};
|
||||
use aura_research::{Axis, CampaignDoc, Cmp, ProcessDoc, SelectRule, StageBlock};
|
||||
|
||||
/// One structural cell of the campaign matrix: (strategy, instrument,
|
||||
@@ -63,9 +63,8 @@ pub enum MemberFault {
|
||||
/// consumer phrases them (the DocFault/RefFault pattern).
|
||||
#[derive(Debug)]
|
||||
pub enum ExecFault {
|
||||
/// v1 boundary: `std::monte_carlo` / `std::generalize` are not executable.
|
||||
UnsupportedStage { stage: usize, block: String },
|
||||
/// The pipeline is not `std::sweep (std::gate)* (std::walk_forward)?`.
|
||||
/// The pipeline is not `std::sweep (std::gate)* (std::walk_forward)?
|
||||
/// (std::monte_carlo)? (std::generalize)?`.
|
||||
PipelineShape { detail: String },
|
||||
/// A sweep/walk_forward selection metric outside the registry's rankable
|
||||
/// roster.
|
||||
@@ -78,6 +77,15 @@ pub enum ExecFault {
|
||||
PlateauInWalkForward { stage: usize },
|
||||
/// sweep `deflate: true` with a non-argmax select rule.
|
||||
DeflatePlateauConflict { stage: usize },
|
||||
/// generalize needs >= 2 instruments in the campaign (static).
|
||||
GeneralizeNeedsInstruments { available: usize },
|
||||
/// generalize's metric must be an R metric (static — the intrinsic tier
|
||||
/// accepts any vocabulary name; the shipped generalization() would only
|
||||
/// refuse at runtime).
|
||||
GeneralizeNonRMetric { metric: String },
|
||||
/// monte_carlo with resamples == 0 or block_len == 0 (static, instead of
|
||||
/// the engine's defined all-zero degenerate).
|
||||
ZeroBootstrapParam { stage: usize, field: &'static str },
|
||||
/// `WindowRoller` construction refusals at runtime.
|
||||
Window { stage: usize, detail: String },
|
||||
Member(MemberFault),
|
||||
@@ -156,46 +164,73 @@ fn predicate_holds(cmp: &Cmp, value: f64, threshold: f64) -> bool {
|
||||
}
|
||||
|
||||
/// Statically refuse everything refusable before any member runs (the F7
|
||||
/// lesson applied forward): the v1 executable pipeline shape is exactly
|
||||
/// `std::sweep (std::gate)* (std::walk_forward)?`; every sweep/walk_forward
|
||||
/// selection metric is in [`RANKABLE_METRICS`]; every gate predicate metric
|
||||
/// is in [`PER_MEMBER_METRICS`]; walk_forward must not select `plateau:*`
|
||||
/// (a gated survivor subset has no grid lattice); sweep `deflate: true`
|
||||
/// composes only with `argmax`; walk_forward lengths must fit `i64` (the
|
||||
/// roller's Timestamp unit). The campaign parameter is the seam for
|
||||
/// campaign-level static checks (none in v1, hence unused).
|
||||
pub(crate) fn preflight(process: &ProcessDoc, _campaign: &CampaignDoc) -> Result<(), ExecFault> {
|
||||
// v1 boundary first: an mc/generalize stage refuses wherever it sits,
|
||||
// before any shape complaint about the same stage.
|
||||
for (i, stage) in process.pipeline.iter().enumerate() {
|
||||
let block = match stage {
|
||||
/// lesson applied forward): the v2 executable pipeline shape is exactly
|
||||
/// `std::sweep (std::gate)* (std::walk_forward)? (std::monte_carlo)?
|
||||
/// (std::generalize)?` — each suffix stage at most once, order fixed,
|
||||
/// `std::generalize` strictly last; every sweep/walk_forward selection
|
||||
/// metric is in [`RANKABLE_METRICS`]; every gate predicate metric is in
|
||||
/// [`PER_MEMBER_METRICS`]; walk_forward must not select `plateau:*` (a gated
|
||||
/// survivor subset has no grid lattice); sweep `deflate: true` composes only
|
||||
/// with `argmax`; walk_forward lengths must fit `i64` (the roller's
|
||||
/// Timestamp unit); monte_carlo `resamples` and `block_len` must be > 0
|
||||
/// (static, instead of the engine's defined all-zero degenerate); generalize
|
||||
/// needs >= 2 campaign instruments and an R selection metric (the registry's
|
||||
/// `check_r_metric`). The campaign parameter carries the campaign-level
|
||||
/// static checks (generalize's instrument arity).
|
||||
pub(crate) fn preflight(process: &ProcessDoc, campaign: &CampaignDoc) -> Result<(), ExecFault> {
|
||||
// Position of a stage in the fixed v2 shape; gates (rank 1) may repeat,
|
||||
// every other rank appears at most once and ranks never decrease.
|
||||
fn shape_rank(stage: &StageBlock) -> usize {
|
||||
match stage {
|
||||
StageBlock::Sweep { .. } => 0,
|
||||
StageBlock::Gate { .. } => 1,
|
||||
StageBlock::WalkForward { .. } => 2,
|
||||
StageBlock::MonteCarlo { .. } => 3,
|
||||
StageBlock::Generalize { .. } => 4,
|
||||
}
|
||||
}
|
||||
fn block_id(stage: &StageBlock) -> &'static str {
|
||||
match stage {
|
||||
StageBlock::Sweep { .. } => "std::sweep",
|
||||
StageBlock::Gate { .. } => "std::gate",
|
||||
StageBlock::WalkForward { .. } => "std::walk_forward",
|
||||
StageBlock::MonteCarlo { .. } => "std::monte_carlo",
|
||||
StageBlock::Generalize { .. } => "std::generalize",
|
||||
_ => continue,
|
||||
};
|
||||
return Err(ExecFault::UnsupportedStage { stage: i, block: block.to_string() });
|
||||
}
|
||||
}
|
||||
// shape: exactly `std::sweep (std::gate)* (std::walk_forward)?`.
|
||||
// shape: `std::sweep (std::gate)* (std::walk_forward)? (std::monte_carlo)?
|
||||
// (std::generalize)?` — a monotone rank walk over adjacent pairs captures
|
||||
// every violation: a rank drop is an out-of-order stage (an annotator
|
||||
// before a population stage, anything after std::generalize), an adjacent
|
||||
// equal rank above the gate tier is a duplicate suffix stage.
|
||||
if !matches!(process.pipeline.first(), Some(StageBlock::Sweep { .. })) {
|
||||
return Err(ExecFault::PipelineShape {
|
||||
detail: "the first stage must be std::sweep".to_string(),
|
||||
});
|
||||
}
|
||||
let last = process.pipeline.len() - 1;
|
||||
let mut prev = &process.pipeline[0];
|
||||
for (i, stage) in process.pipeline.iter().enumerate().skip(1) {
|
||||
match stage {
|
||||
StageBlock::Sweep { .. } => {
|
||||
return Err(ExecFault::PipelineShape {
|
||||
detail: format!("stage {i}: only the first stage may be std::sweep"),
|
||||
});
|
||||
}
|
||||
StageBlock::WalkForward { .. } if i != last => {
|
||||
return Err(ExecFault::PipelineShape {
|
||||
detail: format!("stage {i}: std::walk_forward must be the final stage"),
|
||||
});
|
||||
}
|
||||
_ => {}
|
||||
if matches!(stage, StageBlock::Sweep { .. }) {
|
||||
return Err(ExecFault::PipelineShape {
|
||||
detail: format!("stage {i}: only the first stage may be std::sweep"),
|
||||
});
|
||||
}
|
||||
let (rank, prev_rank) = (shape_rank(stage), shape_rank(prev));
|
||||
if rank < prev_rank {
|
||||
return Err(ExecFault::PipelineShape {
|
||||
detail: format!(
|
||||
"stage {i}: {} cannot follow {}",
|
||||
block_id(stage),
|
||||
block_id(prev)
|
||||
),
|
||||
});
|
||||
}
|
||||
if rank == prev_rank && rank > 1 {
|
||||
return Err(ExecFault::PipelineShape {
|
||||
detail: format!("stage {i}: {} may appear at most once", block_id(stage)),
|
||||
});
|
||||
}
|
||||
prev = stage;
|
||||
}
|
||||
// per-stage slot rules (shape already established above).
|
||||
for (i, stage) in process.pipeline.iter().enumerate() {
|
||||
@@ -244,8 +279,22 @@ pub(crate) fn preflight(process: &ProcessDoc, _campaign: &CampaignDoc) -> Result
|
||||
}
|
||||
}
|
||||
}
|
||||
StageBlock::MonteCarlo { .. } | StageBlock::Generalize { .. } => {
|
||||
unreachable!("refused by the v1 scan above")
|
||||
StageBlock::MonteCarlo { resamples, block_len } => {
|
||||
if *resamples == 0 {
|
||||
return Err(ExecFault::ZeroBootstrapParam { stage: i, field: "resamples" });
|
||||
}
|
||||
if *block_len == 0 {
|
||||
return Err(ExecFault::ZeroBootstrapParam { stage: i, field: "block_len" });
|
||||
}
|
||||
}
|
||||
StageBlock::Generalize { metric } => {
|
||||
let available = campaign.data.instruments.len();
|
||||
if available < 2 {
|
||||
return Err(ExecFault::GeneralizeNeedsInstruments { available });
|
||||
}
|
||||
if check_r_metric(metric).is_err() {
|
||||
return Err(ExecFault::GeneralizeNonRMetric { metric: metric.clone() });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -438,16 +487,36 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn mc_stage(resamples: u32, block_len: u32) -> StageBlock {
|
||||
StageBlock::MonteCarlo { resamples, block_len }
|
||||
}
|
||||
|
||||
fn generalize_stage(metric: &str) -> StageBlock {
|
||||
StageBlock::Generalize { metric: metric.to_string() }
|
||||
}
|
||||
|
||||
/// [`campaign`] with a second instrument — generalize's static arity
|
||||
/// guard needs >= 2 to pass.
|
||||
fn campaign_two_instruments() -> CampaignDoc {
|
||||
let mut c = campaign();
|
||||
c.data.instruments.push("GER40".to_string());
|
||||
c
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preflight_accepts_the_v1_shape() {
|
||||
fn preflight_accepts_the_v2_shapes() {
|
||||
let c = campaign();
|
||||
// the full v1 shape: sweep (gate)* (walk_forward)?
|
||||
let c2 = campaign_two_instruments();
|
||||
// the full v2 shape: sweep (gate)* (walk_forward)? (monte_carlo)?
|
||||
// (generalize)?
|
||||
let full = process_of(vec![
|
||||
sweep_stage("sqn_normalized", SelectRule::Argmax, true),
|
||||
gate_stage("net_expectancy_r"),
|
||||
wf_stage("sqn_normalized", SelectRule::Argmax),
|
||||
mc_stage(1000, 5),
|
||||
generalize_stage("net_expectancy_r"),
|
||||
]);
|
||||
assert!(preflight(&full, &c).is_ok());
|
||||
assert!(preflight(&full, &c2).is_ok());
|
||||
// degenerate accepted shapes: bare sweep (plateau select without
|
||||
// deflate is legal on a sweep); sweep + gates without walk_forward.
|
||||
let bare = process_of(vec![sweep_stage("total_pips", SelectRule::PlateauMean, false)]);
|
||||
@@ -458,26 +527,166 @@ mod tests {
|
||||
gate_stage("win_rate"),
|
||||
]);
|
||||
assert!(preflight(&gated, &c).is_ok());
|
||||
// suffix stages compose independently: mc without wf (single
|
||||
// instrument fine), generalize without mc (needs 2 instruments),
|
||||
// wf + mc without generalize.
|
||||
let mc_only = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
mc_stage(1000, 5),
|
||||
]);
|
||||
assert!(preflight(&mc_only, &c).is_ok());
|
||||
let gen_only = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
generalize_stage("expectancy_r"),
|
||||
]);
|
||||
assert!(preflight(&gen_only, &c2).is_ok());
|
||||
let wf_mc = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
wf_stage("sqn", SelectRule::Argmax),
|
||||
mc_stage(100, 3),
|
||||
]);
|
||||
assert!(preflight(&wf_mc, &c).is_ok());
|
||||
}
|
||||
|
||||
/// v2 shape: an annotator may not precede a population stage, and
|
||||
/// std::generalize is strictly last. `[sweep, mc, walk_forward]` is
|
||||
/// intrinsically valid at the document tier yet refused here.
|
||||
#[test]
|
||||
fn preflight_refuses_mc_and_generalize_stages() {
|
||||
let c = campaign();
|
||||
let mc = process_of(vec![
|
||||
fn preflight_refuses_annotator_order_violations() {
|
||||
let c2 = campaign_two_instruments();
|
||||
let cases: Vec<(Vec<StageBlock>, &str)> = vec![
|
||||
(
|
||||
vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
mc_stage(100, 5),
|
||||
wf_stage("sqn", SelectRule::Argmax),
|
||||
],
|
||||
"stage 2: std::walk_forward cannot follow std::monte_carlo",
|
||||
),
|
||||
(
|
||||
vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
mc_stage(100, 5),
|
||||
gate_stage("expectancy_r"),
|
||||
],
|
||||
"stage 2: std::gate cannot follow std::monte_carlo",
|
||||
),
|
||||
(
|
||||
vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
generalize_stage("expectancy_r"),
|
||||
mc_stage(100, 5),
|
||||
],
|
||||
"stage 2: std::monte_carlo cannot follow std::generalize",
|
||||
),
|
||||
(
|
||||
vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
wf_stage("sqn", SelectRule::Argmax),
|
||||
gate_stage("expectancy_r"),
|
||||
],
|
||||
"stage 2: std::gate cannot follow std::walk_forward",
|
||||
),
|
||||
];
|
||||
for (pipeline, want) in cases {
|
||||
match preflight(&process_of(pipeline), &c2) {
|
||||
Err(ExecFault::PipelineShape { detail }) => assert_eq!(detail, want),
|
||||
other => panic!("expected PipelineShape({want}), got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Each v2 suffix stage appears at most once.
|
||||
#[test]
|
||||
fn preflight_refuses_duplicate_suffix_stages() {
|
||||
let c2 = campaign_two_instruments();
|
||||
let cases: Vec<(Vec<StageBlock>, &str)> = vec![
|
||||
(
|
||||
vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
wf_stage("sqn", SelectRule::Argmax),
|
||||
wf_stage("sqn", SelectRule::Argmax),
|
||||
],
|
||||
"stage 2: std::walk_forward may appear at most once",
|
||||
),
|
||||
(
|
||||
vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
mc_stage(100, 5),
|
||||
mc_stage(100, 5),
|
||||
],
|
||||
"stage 2: std::monte_carlo may appear at most once",
|
||||
),
|
||||
(
|
||||
vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
generalize_stage("expectancy_r"),
|
||||
generalize_stage("expectancy_r"),
|
||||
],
|
||||
"stage 2: std::generalize may appear at most once",
|
||||
),
|
||||
];
|
||||
for (pipeline, want) in cases {
|
||||
match preflight(&process_of(pipeline), &c2) {
|
||||
Err(ExecFault::PipelineShape { detail }) => assert_eq!(detail, want),
|
||||
other => panic!("expected PipelineShape({want}), got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// generalize's static arity guard: the campaign must carry >= 2
|
||||
/// instruments (the shipped generalization() would only refuse at
|
||||
/// runtime, after members ran).
|
||||
#[test]
|
||||
fn preflight_refuses_single_instrument_generalize() {
|
||||
let c = campaign(); // one instrument
|
||||
let p = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
StageBlock::MonteCarlo { resamples: 100, block_len: 5 },
|
||||
generalize_stage("expectancy_r"),
|
||||
]);
|
||||
assert!(matches!(
|
||||
preflight(&mc, &c),
|
||||
Err(ExecFault::UnsupportedStage { stage: 1, block }) if block == "std::monte_carlo"
|
||||
preflight(&p, &c),
|
||||
Err(ExecFault::GeneralizeNeedsInstruments { available: 1 })
|
||||
));
|
||||
let g = process_of(vec![
|
||||
}
|
||||
|
||||
/// generalize's metric guard: check_r_metric refuses pip metrics AND
|
||||
/// unknown names — both surface as GeneralizeNonRMetric.
|
||||
#[test]
|
||||
fn preflight_refuses_non_r_generalize_metric() {
|
||||
let c2 = campaign_two_instruments();
|
||||
for bad in ["total_pips", "no_such_metric"] {
|
||||
let p = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
generalize_stage(bad),
|
||||
]);
|
||||
assert!(matches!(
|
||||
preflight(&p, &c2),
|
||||
Err(ExecFault::GeneralizeNonRMetric { metric }) if metric == bad
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
/// monte_carlo zero params are static refusals (instead of the engine's
|
||||
/// defined all-zero degenerate at runtime), naming the offending field.
|
||||
#[test]
|
||||
fn preflight_refuses_zero_bootstrap_params() {
|
||||
let c = campaign();
|
||||
let zero_resamples = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
StageBlock::Generalize { metric: "expectancy_r".to_string() },
|
||||
mc_stage(0, 5),
|
||||
]);
|
||||
assert!(matches!(
|
||||
preflight(&g, &c),
|
||||
Err(ExecFault::UnsupportedStage { stage: 1, block }) if block == "std::generalize"
|
||||
preflight(&zero_resamples, &c),
|
||||
Err(ExecFault::ZeroBootstrapParam { stage: 1, field: "resamples" })
|
||||
));
|
||||
let zero_block = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
mc_stage(100, 0),
|
||||
]);
|
||||
assert!(matches!(
|
||||
preflight(&zero_block, &c),
|
||||
Err(ExecFault::ZeroBootstrapParam { stage: 1, field: "block_len" })
|
||||
));
|
||||
}
|
||||
|
||||
@@ -493,7 +702,8 @@ mod tests {
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
]);
|
||||
assert!(matches!(preflight(&double, &c), Err(ExecFault::PipelineShape { .. })));
|
||||
// walk_forward anywhere but last is a shape refusal too.
|
||||
// a gate after walk_forward is still a shape refusal (v2 relaxes
|
||||
// wf-final only for the annotator suffix: mc/generalize may follow).
|
||||
let wf_mid = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
wf_stage("sqn", SelectRule::Argmax),
|
||||
|
||||
@@ -44,10 +44,6 @@ pub(crate) fn is_content_id(s: &str) -> bool {
|
||||
/// (stage index + block id), the `doc_fault_prose`/`ref_fault_prose` register.
|
||||
fn exec_fault_prose(f: &ExecFault) -> String {
|
||||
match f {
|
||||
ExecFault::UnsupportedStage { stage, block } => format!(
|
||||
"process stage {stage} ({block}) is not executable in v1; executable \
|
||||
pipeline shape: std::sweep [std::gate]* [std::walk_forward]"
|
||||
),
|
||||
ExecFault::PipelineShape { detail } => {
|
||||
format!("process pipeline is not executable: {detail}")
|
||||
}
|
||||
@@ -68,6 +64,16 @@ fn exec_fault_prose(f: &ExecFault) -> String {
|
||||
ExecFault::DeflatePlateauConflict { stage } => format!(
|
||||
"process stage {stage}: sweep deflate: true composes only with select \"argmax\""
|
||||
),
|
||||
ExecFault::GeneralizeNeedsInstruments { available } => format!(
|
||||
"campaign has {available} instrument(s); std::generalize needs at least 2"
|
||||
),
|
||||
ExecFault::GeneralizeNonRMetric { metric } => format!(
|
||||
"std::generalize metric \"{metric}\" is not an R metric (gross/net R \
|
||||
families generalize; pip metrics do not)"
|
||||
),
|
||||
ExecFault::ZeroBootstrapParam { stage, field } => format!(
|
||||
"process stage {stage}: monte_carlo {field} must be > 0"
|
||||
),
|
||||
ExecFault::Window { stage, detail } => format!(
|
||||
"process stage {stage}: walk_forward windows do not fit the campaign \
|
||||
window: {detail}"
|
||||
@@ -484,6 +490,42 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// generalize's static instrument-arity refusal is Debug-free and
|
||||
/// names the available count plus the required floor.
|
||||
fn generalize_needs_instruments_prose_names_the_shortfall() {
|
||||
let prose = exec_fault_prose(&ExecFault::GeneralizeNeedsInstruments { available: 1 });
|
||||
assert_eq!(prose, "campaign has 1 instrument(s); std::generalize needs at least 2");
|
||||
assert!(!prose.contains("GeneralizeNeedsInstruments"), "Debug leak: {prose}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// generalize's R-metric refusal names the metric and the R-only
|
||||
/// rationale (pip metrics are not comparable across instruments).
|
||||
fn generalize_non_r_metric_prose_names_the_metric() {
|
||||
let prose = exec_fault_prose(&ExecFault::GeneralizeNonRMetric {
|
||||
metric: "total_pips".into(),
|
||||
});
|
||||
assert_eq!(
|
||||
prose,
|
||||
"std::generalize metric \"total_pips\" is not an R metric (gross/net R \
|
||||
families generalize; pip metrics do not)"
|
||||
);
|
||||
assert!(!prose.contains("GeneralizeNonRMetric"), "Debug leak: {prose}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// the zero-bootstrap-param refusal is path-addressed (stage index)
|
||||
/// and names the offending field.
|
||||
fn zero_bootstrap_param_prose_names_stage_and_field() {
|
||||
let prose = exec_fault_prose(&ExecFault::ZeroBootstrapParam {
|
||||
stage: 3,
|
||||
field: "block_len",
|
||||
});
|
||||
assert_eq!(prose, "process stage 3: monte_carlo block_len must be > 0");
|
||||
assert!(!prose.contains("ZeroBootstrapParam"), "Debug leak: {prose}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// The only shape treated as a direct store address is a bare 64-char
|
||||
/// lowercase-hex token; anything else (wrong length, uppercase, non-hex)
|
||||
|
||||
@@ -794,6 +794,153 @@ fn campaign_run_v1_boundary_refuses_mc_process() {
|
||||
assert!(!out.contains("UnsupportedStage"), "Debug leak: {out}");
|
||||
}
|
||||
|
||||
/// A v2 process doc with `std::walk_forward` following the annotator
|
||||
/// `std::monte_carlo` — intrinsically valid (order across population vs.
|
||||
/// annotator stages is not a document-tier concern), refused only by the
|
||||
/// executor's v2 preflight shape guard.
|
||||
const WF_AFTER_MC_PROCESS_DOC: &str = r#"{
|
||||
"format_version": 1,
|
||||
"kind": "process",
|
||||
"name": "wf-after-mc",
|
||||
"pipeline": [
|
||||
{ "block": "std::sweep", "metric": "sqn_normalized", "select": "argmax" },
|
||||
{ "block": "std::monte_carlo", "resamples": 100, "block_len": 5 },
|
||||
{ "block": "std::walk_forward", "in_sample_ms": 1209600000, "out_of_sample_ms": 604800000,
|
||||
"step_ms": 604800000, "mode": "rolling", "metric": "net_expectancy_r", "select": "argmax" }
|
||||
]
|
||||
}"#;
|
||||
|
||||
/// Property (0108 task 2, mirroring aura-research's characterization pin
|
||||
/// `validate_process_permits_walk_forward_after_an_annotator`): the
|
||||
/// intrinsic/executor tier boundary is observable across TWO CLI commands.
|
||||
/// `process validate` accepts `[sweep, monte_carlo, walk_forward]` (exit 0 —
|
||||
/// the document tier does not order population vs. annotator stages) while
|
||||
/// `campaign run`'s v2 preflight refuses the very same document, before any
|
||||
/// member executes (data-free: the [1, 2] 1970-epoch-ms window is never
|
||||
/// reached), because `std::walk_forward` cannot follow the annotator
|
||||
/// `std::monte_carlo`. A regression that let the executor silently accept the
|
||||
/// intrinsic tier's looser order (or that widened the intrinsic tier to
|
||||
/// match) would collapse this two-command divergence.
|
||||
#[test]
|
||||
fn process_validate_permits_wf_after_mc_but_campaign_run_refuses_it() {
|
||||
// intrinsic tier: no project needed, and it accepts the doc outright.
|
||||
let vdir = temp_cwd("wf-after-mc-validate");
|
||||
write_doc(&vdir, "wfmc.process.json", WF_AFTER_MC_PROCESS_DOC);
|
||||
let (vout, vcode) = run_code_in(&vdir, &["process", "validate", "wfmc.process.json"]);
|
||||
assert_eq!(vcode, Some(0), "intrinsic validate must accept sweep->mc->walk_forward: {vout}");
|
||||
assert!(
|
||||
vout.contains("process document valid (intrinsic): 3 pipeline blocks"),
|
||||
"stdout/stderr: {vout}"
|
||||
);
|
||||
|
||||
// executor tier: campaign run's v2 preflight refuses the same shape.
|
||||
let _fixture = project_lock();
|
||||
let dir = built_project();
|
||||
let runs_dir = dir.join("runs");
|
||||
std::fs::remove_dir_all(&runs_dir).ok();
|
||||
let _cleanup = ScratchGuard(vec![
|
||||
ScratchPath::Dir(runs_dir.clone()),
|
||||
ScratchPath::File(dir.join("wfmc.process.json")),
|
||||
ScratchPath::File(dir.join("wfmc.campaign.json")),
|
||||
]);
|
||||
let bp_id = seed_blueprint(dir, "campaign-run-wfmc-seed");
|
||||
let proc_id = register_process_doc(dir, "wfmc.process.json", WF_AFTER_MC_PROCESS_DOC);
|
||||
write_doc(dir, "wfmc.campaign.json", &campaign_doc_json(&bp_id, &proc_id, (1, 2), "", ""));
|
||||
let (out, code) = run_code_in(dir, &["campaign", "run", "wfmc.campaign.json"]);
|
||||
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
||||
assert!(
|
||||
out.contains("std::walk_forward cannot follow std::monte_carlo"),
|
||||
"stdout/stderr: {out}"
|
||||
);
|
||||
assert!(!out.contains("PipelineShape {"), "Debug leak: {out}");
|
||||
}
|
||||
|
||||
/// A v2 process doc with a static `std::monte_carlo resamples: 0` defect.
|
||||
const ZERO_RESAMPLES_MC_PROCESS_DOC: &str = r#"{
|
||||
"format_version": 1,
|
||||
"kind": "process",
|
||||
"name": "zero-resamples-mc",
|
||||
"pipeline": [
|
||||
{ "block": "std::sweep", "metric": "sqn_normalized", "select": "argmax" },
|
||||
{ "block": "std::monte_carlo", "resamples": 0, "block_len": 5 }
|
||||
]
|
||||
}"#;
|
||||
|
||||
/// Property (0108 task 2): `std::monte_carlo`'s `resamples > 0` guard is a
|
||||
/// STATIC executor-preflight refusal, wired end to end from the on-disk
|
||||
/// document through to CLI stdout, firing before any member runs (data-free:
|
||||
/// the [1, 2] 1970-epoch-ms window is never reached) and naming the stage
|
||||
/// index and the offending field. A regression that dropped the static guard
|
||||
/// (deferring to the engine's runtime all-zero-resamples degenerate instead)
|
||||
/// would let this refusal disappear or drift to a different message.
|
||||
#[test]
|
||||
fn campaign_run_refuses_zero_resamples_monte_carlo_before_any_member_runs() {
|
||||
let _fixture = project_lock();
|
||||
let dir = built_project();
|
||||
let runs_dir = dir.join("runs");
|
||||
std::fs::remove_dir_all(&runs_dir).ok();
|
||||
let _cleanup = ScratchGuard(vec![
|
||||
ScratchPath::Dir(runs_dir.clone()),
|
||||
ScratchPath::File(dir.join("zeromc.process.json")),
|
||||
ScratchPath::File(dir.join("zeromc.campaign.json")),
|
||||
]);
|
||||
let bp_id = seed_blueprint(dir, "campaign-run-zeromc-seed");
|
||||
let proc_id = register_process_doc(dir, "zeromc.process.json", ZERO_RESAMPLES_MC_PROCESS_DOC);
|
||||
write_doc(dir, "zeromc.campaign.json", &campaign_doc_json(&bp_id, &proc_id, (1, 2), "", ""));
|
||||
let (out, code) = run_code_in(dir, &["campaign", "run", "zeromc.campaign.json"]);
|
||||
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
||||
assert!(
|
||||
out.contains("process stage 1: monte_carlo resamples must be > 0"),
|
||||
"stdout/stderr: {out}"
|
||||
);
|
||||
assert!(!out.contains("ZeroBootstrapParam"), "Debug leak: {out}");
|
||||
}
|
||||
|
||||
/// A v2 process doc that ends in `std::generalize`.
|
||||
const GENERALIZE_PROCESS_DOC: &str = r#"{
|
||||
"format_version": 1,
|
||||
"kind": "process",
|
||||
"name": "generalize-screen",
|
||||
"pipeline": [
|
||||
{ "block": "std::sweep", "metric": "sqn_normalized", "select": "argmax" },
|
||||
{ "block": "std::generalize", "metric": "net_expectancy_r" }
|
||||
]
|
||||
}"#;
|
||||
|
||||
/// Property (0108 task 2): `std::generalize`'s campaign-level instrument-
|
||||
/// arity guard (>= 2 instruments) is a STATIC executor-preflight refusal,
|
||||
/// wired end to end from the campaign document's `data.instruments` through
|
||||
/// to CLI stdout, firing before any member runs (data-free) and naming the
|
||||
/// actual instrument count. `campaign_doc_json`'s fixture campaign carries
|
||||
/// exactly one instrument (`GER40`), so this exercises the campaign
|
||||
/// parameter `preflight` newly reads (previously unused in v1) rather than
|
||||
/// anything reachable from the process document alone. A regression that
|
||||
/// left the campaign parameter unread would silently accept a single-
|
||||
/// instrument `std::generalize` and only fail later, opaquely, inside the
|
||||
/// generalization math.
|
||||
#[test]
|
||||
fn campaign_run_refuses_single_instrument_generalize() {
|
||||
let _fixture = project_lock();
|
||||
let dir = built_project();
|
||||
let runs_dir = dir.join("runs");
|
||||
std::fs::remove_dir_all(&runs_dir).ok();
|
||||
let _cleanup = ScratchGuard(vec![
|
||||
ScratchPath::Dir(runs_dir.clone()),
|
||||
ScratchPath::File(dir.join("gen.process.json")),
|
||||
ScratchPath::File(dir.join("gen.campaign.json")),
|
||||
]);
|
||||
let bp_id = seed_blueprint(dir, "campaign-run-gen-seed");
|
||||
let proc_id = register_process_doc(dir, "gen.process.json", GENERALIZE_PROCESS_DOC);
|
||||
write_doc(dir, "gen.campaign.json", &campaign_doc_json(&bp_id, &proc_id, (1, 2), "", ""));
|
||||
let (out, code) = run_code_in(dir, &["campaign", "run", "gen.campaign.json"]);
|
||||
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
||||
assert!(
|
||||
out.contains("campaign has 1 instrument(s); std::generalize needs at least 2"),
|
||||
"stdout/stderr: {out}"
|
||||
);
|
||||
assert!(!out.contains("GeneralizeNeedsInstruments"), "Debug leak: {out}");
|
||||
}
|
||||
|
||||
/// Non-empty persist_taps defers LOUDLY, and the note's ORDER is pinned: it
|
||||
/// prints before member execution, so it is asserted here on a run
|
||||
/// that refuses at the member-data seam. The [1, 2] epoch-ms window (1970)
|
||||
|
||||
@@ -157,7 +157,7 @@ where
|
||||
/// (#139). `block_len == 1` is the i.i.d. trade-shuffle; `block_len > 1` resamples
|
||||
/// contiguous runs, preserving the serial correlation of sequential trades a pure
|
||||
/// shuffle would erase. Deterministic (C1) given `seed`.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct RBootstrap {
|
||||
pub e_r: MetricStats, // mean + p5/p25/p50/p75/p95 of the resampled E[R]
|
||||
pub prob_le_zero: f64, // fraction of resamples whose mean R <= 0
|
||||
|
||||
@@ -30,8 +30,8 @@ mod compat;
|
||||
mod lineage;
|
||||
pub use lineage::{
|
||||
group_families, mc_member_reports, sweep_member_reports, walkforward_member_reports,
|
||||
CampaignRunRecord, CellRealization, Family, FamilyKind, FamilyRunRecord, StageRealization,
|
||||
StageSelection,
|
||||
CampaignGeneralization, CampaignRunRecord, CellRealization, Family, FamilyKind,
|
||||
FamilyRunRecord, StageBootstrap, StageRealization, StageSelection,
|
||||
};
|
||||
|
||||
mod trace_store;
|
||||
@@ -661,7 +661,7 @@ pub fn optimize_deflated(
|
||||
/// order. `worst_case` is `min_i metric_i`; since every R-metric is
|
||||
/// higher-is-better, the min is unconditionally the conservative floor (no
|
||||
/// direction branch, unlike `PlateauMode::Worst`).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Generalization {
|
||||
pub selection_metric: String,
|
||||
pub n_instruments: usize,
|
||||
@@ -1710,6 +1710,7 @@ mod tests {
|
||||
run: 99,
|
||||
seed: 7,
|
||||
cells: vec![],
|
||||
generalizations: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1777,18 +1778,116 @@ mod tests {
|
||||
n_neighbours: None,
|
||||
},
|
||||
}),
|
||||
bootstrap: None,
|
||||
},
|
||||
StageRealization {
|
||||
block: "std::gate".to_string(),
|
||||
family_id: None,
|
||||
survivor_ordinals: Some(vec![0, 3, 4, 7]),
|
||||
selection: None,
|
||||
bootstrap: None,
|
||||
},
|
||||
],
|
||||
}],
|
||||
generalizations: vec![],
|
||||
};
|
||||
let run = reg.append_campaign_run(&record).expect("append");
|
||||
assert_eq!(run, 0);
|
||||
assert_eq!(reg.load_campaign_runs().expect("load"), vec![record]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn campaign_run_record_roundtrips_with_annotator_fields() {
|
||||
use aura_engine::r_bootstrap;
|
||||
let path = temp_family_dir("campaign_run_annotator_roundtrip");
|
||||
let reg = Registry::open(&path);
|
||||
// r_bootstrap is pure given its seed (C1), and serde_json round-trips
|
||||
// finite f64 exactly, so whole-record PartialEq holds across the store.
|
||||
let per_a = r_bootstrap(&[0.5, -0.2, 0.3, 1.1], 8, 2, 7);
|
||||
let per_b = r_bootstrap(&[-0.4, 0.9], 8, 1, 7);
|
||||
let pooled = r_bootstrap(&[0.2, 0.2, -0.1, 0.6, 0.4], 8, 2, 7);
|
||||
let record = CampaignRunRecord {
|
||||
campaign: "feed".to_string(),
|
||||
process: "beef".to_string(),
|
||||
run: 0, // matches the counter's first assignment, so whole-record PartialEq holds
|
||||
seed: 7,
|
||||
cells: vec![
|
||||
CellRealization {
|
||||
strategy: "3f9c".to_string(),
|
||||
instrument: "EURUSD".to_string(),
|
||||
window_ms: (0, 1000),
|
||||
stages: vec![StageRealization {
|
||||
block: "std::monte_carlo".to_string(),
|
||||
family_id: None,
|
||||
survivor_ordinals: None,
|
||||
selection: None,
|
||||
bootstrap: Some(StageBootstrap::PerSurvivor(vec![
|
||||
(0, per_a),
|
||||
(2, per_b),
|
||||
])),
|
||||
}],
|
||||
},
|
||||
CellRealization {
|
||||
strategy: "3f9c".to_string(),
|
||||
instrument: "GER40".to_string(),
|
||||
window_ms: (0, 1000),
|
||||
stages: vec![StageRealization {
|
||||
block: "std::monte_carlo".to_string(),
|
||||
family_id: None,
|
||||
survivor_ordinals: None,
|
||||
selection: None,
|
||||
bootstrap: Some(StageBootstrap::PooledOos(pooled)),
|
||||
}],
|
||||
},
|
||||
],
|
||||
generalizations: vec![CampaignGeneralization {
|
||||
strategy_ordinal: 0,
|
||||
window_ordinal: 0,
|
||||
generalization: Some(Generalization {
|
||||
selection_metric: "net_expectancy_r".to_string(),
|
||||
n_instruments: 2,
|
||||
worst_case: 0.04,
|
||||
sign_agreement: 2,
|
||||
per_instrument: vec![
|
||||
("EURUSD".to_string(), 0.04),
|
||||
("GER40".to_string(), 0.11),
|
||||
],
|
||||
}),
|
||||
winners: vec![
|
||||
(
|
||||
"EURUSD".to_string(),
|
||||
vec![("sma_cross.fast.length".to_string(), Scalar::i64(3))],
|
||||
),
|
||||
(
|
||||
"GER40".to_string(),
|
||||
vec![("sma_cross.fast.length".to_string(), Scalar::i64(5))],
|
||||
),
|
||||
],
|
||||
missing: vec!["US500".to_string()],
|
||||
}],
|
||||
};
|
||||
let run = reg.append_campaign_run(&record).expect("append annotator record");
|
||||
assert_eq!(run, 0);
|
||||
assert_eq!(
|
||||
reg.load_campaign_runs().expect("load annotator record"),
|
||||
vec![record],
|
||||
"bootstrap (both variants) and generalizations survive the round-trip",
|
||||
);
|
||||
}
|
||||
|
||||
/// A pre-0108 stored line — no `bootstrap`, no `generalizations` — still
|
||||
/// parses (the C14/C23 serde-default widening convention): existing
|
||||
/// `campaign_runs.jsonl` stores survive the record widening unchanged.
|
||||
#[test]
|
||||
fn campaign_run_line_without_new_fields_still_parses() {
|
||||
let path = temp_family_dir("campaign_run_pre_widening");
|
||||
let store = path.with_file_name("campaign_runs.jsonl");
|
||||
let line = r#"{"campaign":"cafe","process":"beef","run":0,"seed":7,"cells":[{"strategy":"3f9c","instrument":"EURUSD","window_ms":[0,1000],"stages":[{"block":"std::gate","survivor_ordinals":[0,2]}]}]}"#;
|
||||
fs::write(&store, format!("{line}\n")).expect("write pre-widening line");
|
||||
let reg = Registry::open(&path);
|
||||
let loaded = reg.load_campaign_runs().expect("pre-widening line parses");
|
||||
assert_eq!(loaded.len(), 1);
|
||||
assert_eq!(loaded[0].cells[0].stages[0].bootstrap, None);
|
||||
assert!(loaded[0].generalizations.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,10 +19,12 @@ use std::fs;
|
||||
use std::io::Write;
|
||||
|
||||
use aura_core::Scalar;
|
||||
use aura_engine::{FamilySelection, McFamily, RunReport, SweepFamily, WalkForwardResult};
|
||||
use aura_engine::{
|
||||
FamilySelection, McFamily, RBootstrap, RunReport, SweepFamily, WalkForwardResult,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{Registry, RegistryError};
|
||||
use crate::{Generalization, Registry, RegistryError};
|
||||
|
||||
/// Which C12 orchestration axis produced a family.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -80,6 +82,11 @@ pub struct CampaignRunRecord {
|
||||
pub run: usize,
|
||||
pub seed: u64,
|
||||
pub cells: Vec<CellRealization>,
|
||||
/// Campaign-scope generalize annotations (cycle 0108), one per
|
||||
/// (strategy, window) across instruments. Absent pre-0108 — the
|
||||
/// serde-default widening keeps existing stored lines parseable.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub generalizations: Vec<CampaignGeneralization>,
|
||||
}
|
||||
|
||||
/// One realized (strategy, instrument, window) cell: the pipeline prefix that
|
||||
@@ -108,6 +115,10 @@ pub struct StageRealization {
|
||||
pub survivor_ordinals: Option<Vec<usize>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selection: Option<StageSelection>,
|
||||
/// std::monte_carlo stages only (cycle 0108). Absent pre-0108 — the
|
||||
/// serde-default widening keeps existing stored lines parseable.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap: Option<StageBootstrap>,
|
||||
}
|
||||
|
||||
/// The recorded winner of a selection-bearing stage: its ordinal in the stage's
|
||||
@@ -119,6 +130,38 @@ pub struct StageSelection {
|
||||
pub selection: FamilySelection,
|
||||
}
|
||||
|
||||
/// The recorded result of a `std::monte_carlo` stage (cycle 0108, #200 d1/d4):
|
||||
/// the payload mirrors the stage's input shape.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum StageBootstrap {
|
||||
/// After sweep/gates: one bootstrap per surviving member,
|
||||
/// `(ordinal into the population family, result)`.
|
||||
PerSurvivor(Vec<(usize, RBootstrap)>),
|
||||
/// After a walk_forward: one bootstrap over the pooled per-window
|
||||
/// OOS trade-R series (roll order).
|
||||
PooledOos(RBootstrap),
|
||||
}
|
||||
|
||||
/// The campaign-scope generalize annotation (#200 d2): recorded at the scope
|
||||
/// it is computed, keyed (strategy, window) across instruments.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct CampaignGeneralization {
|
||||
pub strategy_ordinal: usize,
|
||||
pub window_ordinal: usize,
|
||||
/// None when fewer than 2 instrument cells produced a winner (gate
|
||||
/// truncation) — the shortfall is recorded, never computed around.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub generalization: Option<Generalization>,
|
||||
/// Per-instrument winning params — divergent winners are exposed,
|
||||
/// never averaged away. Present whenever >= 1 winner exists.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub winners: Vec<(String, Vec<(String, Scalar)>)>,
|
||||
/// Instrument cells that contributed no winner (gate-truncated), by name.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub missing: Vec<String>,
|
||||
}
|
||||
|
||||
impl Registry {
|
||||
/// Assign a fresh per-name `run` index (one past the highest `run` already
|
||||
/// stored for `name`, or `0` if unseen), stamp `reports` as ordinal-ordered
|
||||
|
||||
@@ -73,8 +73,10 @@ fn campaign_run(campaign: &str) -> CampaignRunRecord {
|
||||
n_neighbours: None,
|
||||
},
|
||||
}),
|
||||
bootstrap: None,
|
||||
}],
|
||||
}],
|
||||
generalizations: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user