From 38ffe50b57c972e6ffd6cce335aaa6194477d31c Mon Sep 17 00:00:00 2001 From: Brummel Date: Sat, 4 Jul 2026 02:08:28 +0200 Subject: [PATCH] feat(cli): campaign validate runs the executor preflight as a third tier (fieldtest 0108 F6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit valid == runnable for every pure document/campaign property: inside a project, once intrinsic + referential pass, validate fetches the referenced process from the store and runs the data-free aura_campaign::preflight (promoted to pub) — an executable shape gains 'campaign document valid (executable): pipeline shape and static guards pass'; a shape/param/metric/instrument fault refuses under 'campaign is not executable:' with the exec_fault_prose detail, exit 1, before any member runs. Outside a project the skip line is unchanged (the third tier needs the store like the second). Collateral the RED author predicted: the referential e2e's OK fixture (PROCESS_DOC: deflate+plateau sweep, overfit_probability gate) was intrinsically-valid-but-never-executable — that test now registers an executable twin (argmax/no-deflate, per-member gate metric); the base PROCESS_DOC fixture stays untouched for all other consumers. RED-first (tdd-author handoff): the third-tier pin observed failing on the two-line validate output. Gates: workspace 1026/0, clippy -D warnings clean. closes #205 --- crates/aura-campaign/src/lib.rs | 2 +- crates/aura-cli/src/campaign_run.rs | 2 +- crates/aura-cli/src/research_docs.rs | 23 +++++++ crates/aura-cli/tests/research_docs.rs | 90 +++++++++++++++++++++++++- 4 files changed, 113 insertions(+), 4 deletions(-) diff --git a/crates/aura-campaign/src/lib.rs b/crates/aura-campaign/src/lib.rs index 6db1531..9ea2c50 100644 --- a/crates/aura-campaign/src/lib.rs +++ b/crates/aura-campaign/src/lib.rs @@ -177,7 +177,7 @@ fn predicate_holds(cmp: &Cmp, value: f64, threshold: f64) -> bool { /// needs >= 2 campaign instruments and an R selection metric (the registry's /// `check_r_metric`). The campaign parameter carries the campaign-level /// static checks (generalize's instrument arity). -pub(crate) fn preflight(process: &ProcessDoc, campaign: &CampaignDoc) -> Result<(), ExecFault> { +pub fn preflight(process: &ProcessDoc, campaign: &CampaignDoc) -> Result<(), ExecFault> { // Position of a stage in the fixed v2 shape; gates (rank 1) may repeat, // every other rank appears at most once and ranks never decrease. fn shape_rank(stage: &StageBlock) -> usize { diff --git a/crates/aura-cli/src/campaign_run.rs b/crates/aura-cli/src/campaign_run.rs index 0365bc4..619e98f 100644 --- a/crates/aura-cli/src/campaign_run.rs +++ b/crates/aura-cli/src/campaign_run.rs @@ -42,7 +42,7 @@ pub(crate) fn is_content_id(s: &str) -> bool { /// Phrase an [`ExecFault`] for stderr — Debug-leak-free, path-addressed /// (stage index + block id), the `doc_fault_prose`/`ref_fault_prose` register. -fn exec_fault_prose(f: &ExecFault) -> String { +pub(crate) fn exec_fault_prose(f: &ExecFault) -> String { match f { ExecFault::PipelineShape { detail } => { format!("process pipeline is not executable: {detail}") diff --git a/crates/aura-cli/src/research_docs.rs b/crates/aura-cli/src/research_docs.rs index 9394cad..fd49651 100644 --- a/crates/aura-cli/src/research_docs.rs +++ b/crates/aura-cli/src/research_docs.rs @@ -375,6 +375,29 @@ fn validate_campaign_file(file: &PathBuf, env: &Env) -> Result<(), String> { )); } println!("campaign document valid (referential): all references resolve, axes are in the param space"); + // Third tier (#205): the executor's data-free static preflight, so + // "valid" means "runnable" for every pure document/campaign property. + // The referential tier just proved the process ref resolves, so the + // fetch cannot miss short of a store race. + let proc_id = match &doc.process.r#ref { + aura_research::DocRef::ContentId(id) | aura_research::DocRef::IdentityId(id) => { + id.clone() + } + }; + let proc_text = env + .registry() + .get_process(&proc_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("no process {proc_id} in the project store"))?; + let process = parse_process(&proc_text) + .map_err(|e| doc_error_prose("stored process document", &e))?; + if let Err(f) = aura_campaign::preflight(&process, &doc) { + return Err(fault_block( + "campaign is not executable:", + vec![crate::campaign_run::exec_fault_prose(&f)], + )); + } + println!("campaign document valid (executable): pipeline shape and static guards pass"); } else { let cwd = std::env::current_dir() .map(|d| d.display().to_string()) diff --git a/crates/aura-cli/tests/research_docs.rs b/crates/aura-cli/tests/research_docs.rs index bb294fd..a39f7b5 100644 --- a/crates/aura-cli/tests/research_docs.rs +++ b/crates/aura-cli/tests/research_docs.rs @@ -370,8 +370,22 @@ fn campaign_validate_in_project_reports_referential_tier_end_to_end() { .to_string_lossy() .into_owned(); - // Register a valid process into the same project store. - write_doc(dir, "seed.process.json", PROCESS_DOC); + // Register a valid AND executable process into the same project store: + // this test's OK campaign must pass all three validate tiers, and + // PROCESS_DOC's sweep (deflate: true + plateau:worst) is intrinsically + // valid but statically refused by the executor preflight (#205's third + // tier) — so the referential fixture gets an argmax/no-deflate twin. + let exec_process = PROCESS_DOC + .replacen( + "\"select\": \"plateau:worst\", \"deflate\": true", + "\"select\": \"argmax\", \"deflate\": false", + 1, + ) + // ... and its gate predicates over the annotation metric + // overfit_probability (not per-member) flip to a per-member scalar. + .replacen("\"overfit_probability\", \"cmp\": \"lt\", \"value\": 0.1", "\"win_rate\", \"cmp\": \"lt\", \"value\": 1.1", 1); + assert_ne!(exec_process, PROCESS_DOC, "replacen must match the sweep stage"); + write_doc(dir, "seed.process.json", &exec_process); let (proc_out, proc_code) = run_code_in(dir, &["process", "register", "seed.process.json"]); assert_eq!(proc_code, Some(0), "seed process register failed: {proc_out}"); let proc_id = proc_out @@ -995,6 +1009,78 @@ fn process_validate_permits_wf_after_mc_but_campaign_run_refuses_it() { assert!(!out.contains("PipelineShape {"), "Debug leak: {out}"); } +/// Property (#205, fieldtest 0108 F6): `aura campaign validate` runs the +/// executor's data-free static preflight (`aura_campaign::preflight`) as a +/// THIRD tier, so a pure process-shape fault that needs no data is caught at +/// validate time, not only at `campaign run` time — "valid" now means +/// "runnable". Two faces of the one behaviour, both inside a project (the +/// tier needs the store to fetch the referenced process, like the referential +/// tier above it): +/// - an executable pipeline shape gains a third line, `campaign document +/// valid (executable): pipeline shape and static guards pass`, exit 0; +/// - a `[sweep, monte_carlo, walk_forward]` process — intrinsically valid +/// (`process register` accepts it) and referentially valid (its ref +/// resolves) yet not an executable shape — refuses at VALIDATE with the +/// executor fault header `campaign is not executable:` and the +/// `exec_fault_prose` shape detail, exit 1, before any member runs. +/// +/// A regression that dropped the third tier would let an executor-unrunnable +/// shape validate clean (exit 0), the F6 friction. +#[test] +fn campaign_validate_runs_the_executor_preflight_as_a_third_tier() { + let _fixture = project_lock(); + let dir = built_project(); + let runs_dir = dir.join("runs"); + std::fs::remove_dir_all(&runs_dir).ok(); + let _cleanup = ScratchGuard(vec![ + ScratchPath::Dir(runs_dir.clone()), + ScratchPath::File(dir.join("runnable.process.json")), + ScratchPath::File(dir.join("runnable.campaign.json")), + ScratchPath::File(dir.join("wfmc.process.json")), + ScratchPath::File(dir.join("wfmc.campaign.json")), + ]); + let bp_id = seed_blueprint(dir, "campaign-validate-3tier-seed"); + + // Face 1 — the happy path: an executable pipeline shape (a bare + // std::sweep). Intrinsic + referential pass today (exit 0), but the + // executor tier is invisible; after #205 validate gains the third + // "valid (executable)" line while staying exit 0. + let ok_proc = register_process_doc(dir, "runnable.process.json", SWEEP_ONLY_PROCESS_DOC); + write_doc(dir, "runnable.campaign.json", &campaign_doc_json(&bp_id, &ok_proc, (1, 2), "", "")); + let (ok_out, ok_code) = run_code_in(dir, &["campaign", "validate", "runnable.campaign.json"]); + assert_eq!(ok_code, Some(0), "an executable campaign validates clean: {ok_out}"); + assert!( + ok_out.contains("campaign document valid (referential): all references resolve"), + "the referential tier line survives the added third tier: {ok_out}" + ); + assert!( + ok_out.contains( + "campaign document valid (executable): pipeline shape and static guards pass" + ), + "the new third (executor-preflight) tier line is emitted: {ok_out}" + ); + + // Face 2 — the refuse path: `[sweep, monte_carlo, walk_forward]` is + // intrinsically valid (register exits 0, asserted in register_process_doc) + // and referentially valid (same seeded blueprint + axes), so validate + // reaches the third tier — which refuses the non-executable shape at + // validate time (data-free: the [1, 2] 1970-epoch-ms window is never + // reached), exit 1, with the executor fault header + the shape prose. + let bad_proc = register_process_doc(dir, "wfmc.process.json", WF_AFTER_MC_PROCESS_DOC); + write_doc(dir, "wfmc.campaign.json", &campaign_doc_json(&bp_id, &bad_proc, (1, 2), "", "")); + let (bad_out, bad_code) = run_code_in(dir, &["campaign", "validate", "wfmc.campaign.json"]); + assert_eq!(bad_code, Some(1), "an executor-unrunnable shape refuses at validate: {bad_out}"); + assert!( + bad_out.contains("campaign is not executable:"), + "the executor-tier fault header (mirrors 'campaign references do not resolve:'): {bad_out}" + ); + assert!( + bad_out.contains("std::walk_forward cannot follow std::monte_carlo"), + "the shape prose names the ordering violation via exec_fault_prose: {bad_out}" + ); + assert!(!bad_out.contains("PipelineShape"), "Debug leak: {bad_out}"); +} + /// A v2 process doc with a static `std::monte_carlo resamples: 0` defect. const ZERO_RESAMPLES_MC_PROCESS_DOC: &str = r#"{ "format_version": 1,