feat(cli): campaign validate runs the executor preflight as a third tier (fieldtest 0108 F6)
valid == runnable for every pure document/campaign property: inside a project, once intrinsic + referential pass, validate fetches the referenced process from the store and runs the data-free aura_campaign::preflight (promoted to pub) — an executable shape gains 'campaign document valid (executable): pipeline shape and static guards pass'; a shape/param/metric/instrument fault refuses under 'campaign is not executable:' with the exec_fault_prose detail, exit 1, before any member runs. Outside a project the skip line is unchanged (the third tier needs the store like the second). Collateral the RED author predicted: the referential e2e's OK fixture (PROCESS_DOC: deflate+plateau sweep, overfit_probability gate) was intrinsically-valid-but-never-executable — that test now registers an executable twin (argmax/no-deflate, per-member gate metric); the base PROCESS_DOC fixture stays untouched for all other consumers. RED-first (tdd-author handoff): the third-tier pin observed failing on the two-line validate output. Gates: workspace 1026/0, clippy -D warnings clean. closes #205
This commit is contained in:
@@ -177,7 +177,7 @@ fn predicate_holds(cmp: &Cmp, value: f64, threshold: f64) -> bool {
|
||||
/// needs >= 2 campaign instruments and an R selection metric (the registry's
|
||||
/// `check_r_metric`). The campaign parameter carries the campaign-level
|
||||
/// static checks (generalize's instrument arity).
|
||||
pub(crate) fn preflight(process: &ProcessDoc, campaign: &CampaignDoc) -> Result<(), ExecFault> {
|
||||
pub fn preflight(process: &ProcessDoc, campaign: &CampaignDoc) -> Result<(), ExecFault> {
|
||||
// Position of a stage in the fixed v2 shape; gates (rank 1) may repeat,
|
||||
// every other rank appears at most once and ranks never decrease.
|
||||
fn shape_rank(stage: &StageBlock) -> usize {
|
||||
|
||||
@@ -42,7 +42,7 @@ pub(crate) fn is_content_id(s: &str) -> bool {
|
||||
|
||||
/// Phrase an [`ExecFault`] for stderr — Debug-leak-free, path-addressed
|
||||
/// (stage index + block id), the `doc_fault_prose`/`ref_fault_prose` register.
|
||||
fn exec_fault_prose(f: &ExecFault) -> String {
|
||||
pub(crate) fn exec_fault_prose(f: &ExecFault) -> String {
|
||||
match f {
|
||||
ExecFault::PipelineShape { detail } => {
|
||||
format!("process pipeline is not executable: {detail}")
|
||||
|
||||
@@ -375,6 +375,29 @@ fn validate_campaign_file(file: &PathBuf, env: &Env) -> Result<(), String> {
|
||||
));
|
||||
}
|
||||
println!("campaign document valid (referential): all references resolve, axes are in the param space");
|
||||
// Third tier (#205): the executor's data-free static preflight, so
|
||||
// "valid" means "runnable" for every pure document/campaign property.
|
||||
// The referential tier just proved the process ref resolves, so the
|
||||
// fetch cannot miss short of a store race.
|
||||
let proc_id = match &doc.process.r#ref {
|
||||
aura_research::DocRef::ContentId(id) | aura_research::DocRef::IdentityId(id) => {
|
||||
id.clone()
|
||||
}
|
||||
};
|
||||
let proc_text = env
|
||||
.registry()
|
||||
.get_process(&proc_id)
|
||||
.map_err(|e| e.to_string())?
|
||||
.ok_or_else(|| format!("no process {proc_id} in the project store"))?;
|
||||
let process = parse_process(&proc_text)
|
||||
.map_err(|e| doc_error_prose("stored process document", &e))?;
|
||||
if let Err(f) = aura_campaign::preflight(&process, &doc) {
|
||||
return Err(fault_block(
|
||||
"campaign is not executable:",
|
||||
vec![crate::campaign_run::exec_fault_prose(&f)],
|
||||
));
|
||||
}
|
||||
println!("campaign document valid (executable): pipeline shape and static guards pass");
|
||||
} else {
|
||||
let cwd = std::env::current_dir()
|
||||
.map(|d| d.display().to_string())
|
||||
|
||||
@@ -370,8 +370,22 @@ fn campaign_validate_in_project_reports_referential_tier_end_to_end() {
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
|
||||
// Register a valid process into the same project store.
|
||||
write_doc(dir, "seed.process.json", PROCESS_DOC);
|
||||
// Register a valid AND executable process into the same project store:
|
||||
// this test's OK campaign must pass all three validate tiers, and
|
||||
// PROCESS_DOC's sweep (deflate: true + plateau:worst) is intrinsically
|
||||
// valid but statically refused by the executor preflight (#205's third
|
||||
// tier) — so the referential fixture gets an argmax/no-deflate twin.
|
||||
let exec_process = PROCESS_DOC
|
||||
.replacen(
|
||||
"\"select\": \"plateau:worst\", \"deflate\": true",
|
||||
"\"select\": \"argmax\", \"deflate\": false",
|
||||
1,
|
||||
)
|
||||
// ... and its gate predicates over the annotation metric
|
||||
// overfit_probability (not per-member) flip to a per-member scalar.
|
||||
.replacen("\"overfit_probability\", \"cmp\": \"lt\", \"value\": 0.1", "\"win_rate\", \"cmp\": \"lt\", \"value\": 1.1", 1);
|
||||
assert_ne!(exec_process, PROCESS_DOC, "replacen must match the sweep stage");
|
||||
write_doc(dir, "seed.process.json", &exec_process);
|
||||
let (proc_out, proc_code) = run_code_in(dir, &["process", "register", "seed.process.json"]);
|
||||
assert_eq!(proc_code, Some(0), "seed process register failed: {proc_out}");
|
||||
let proc_id = proc_out
|
||||
@@ -995,6 +1009,78 @@ fn process_validate_permits_wf_after_mc_but_campaign_run_refuses_it() {
|
||||
assert!(!out.contains("PipelineShape {"), "Debug leak: {out}");
|
||||
}
|
||||
|
||||
/// Property (#205, fieldtest 0108 F6): `aura campaign validate` runs the
|
||||
/// executor's data-free static preflight (`aura_campaign::preflight`) as a
|
||||
/// THIRD tier, so a pure process-shape fault that needs no data is caught at
|
||||
/// validate time, not only at `campaign run` time — "valid" now means
|
||||
/// "runnable". Two faces of the one behaviour, both inside a project (the
|
||||
/// tier needs the store to fetch the referenced process, like the referential
|
||||
/// tier above it):
|
||||
/// - an executable pipeline shape gains a third line, `campaign document
|
||||
/// valid (executable): pipeline shape and static guards pass`, exit 0;
|
||||
/// - a `[sweep, monte_carlo, walk_forward]` process — intrinsically valid
|
||||
/// (`process register` accepts it) and referentially valid (its ref
|
||||
/// resolves) yet not an executable shape — refuses at VALIDATE with the
|
||||
/// executor fault header `campaign is not executable:` and the
|
||||
/// `exec_fault_prose` shape detail, exit 1, before any member runs.
|
||||
///
|
||||
/// A regression that dropped the third tier would let an executor-unrunnable
|
||||
/// shape validate clean (exit 0), the F6 friction.
|
||||
#[test]
|
||||
fn campaign_validate_runs_the_executor_preflight_as_a_third_tier() {
|
||||
let _fixture = project_lock();
|
||||
let dir = built_project();
|
||||
let runs_dir = dir.join("runs");
|
||||
std::fs::remove_dir_all(&runs_dir).ok();
|
||||
let _cleanup = ScratchGuard(vec![
|
||||
ScratchPath::Dir(runs_dir.clone()),
|
||||
ScratchPath::File(dir.join("runnable.process.json")),
|
||||
ScratchPath::File(dir.join("runnable.campaign.json")),
|
||||
ScratchPath::File(dir.join("wfmc.process.json")),
|
||||
ScratchPath::File(dir.join("wfmc.campaign.json")),
|
||||
]);
|
||||
let bp_id = seed_blueprint(dir, "campaign-validate-3tier-seed");
|
||||
|
||||
// Face 1 — the happy path: an executable pipeline shape (a bare
|
||||
// std::sweep). Intrinsic + referential pass today (exit 0), but the
|
||||
// executor tier is invisible; after #205 validate gains the third
|
||||
// "valid (executable)" line while staying exit 0.
|
||||
let ok_proc = register_process_doc(dir, "runnable.process.json", SWEEP_ONLY_PROCESS_DOC);
|
||||
write_doc(dir, "runnable.campaign.json", &campaign_doc_json(&bp_id, &ok_proc, (1, 2), "", ""));
|
||||
let (ok_out, ok_code) = run_code_in(dir, &["campaign", "validate", "runnable.campaign.json"]);
|
||||
assert_eq!(ok_code, Some(0), "an executable campaign validates clean: {ok_out}");
|
||||
assert!(
|
||||
ok_out.contains("campaign document valid (referential): all references resolve"),
|
||||
"the referential tier line survives the added third tier: {ok_out}"
|
||||
);
|
||||
assert!(
|
||||
ok_out.contains(
|
||||
"campaign document valid (executable): pipeline shape and static guards pass"
|
||||
),
|
||||
"the new third (executor-preflight) tier line is emitted: {ok_out}"
|
||||
);
|
||||
|
||||
// Face 2 — the refuse path: `[sweep, monte_carlo, walk_forward]` is
|
||||
// intrinsically valid (register exits 0, asserted in register_process_doc)
|
||||
// and referentially valid (same seeded blueprint + axes), so validate
|
||||
// reaches the third tier — which refuses the non-executable shape at
|
||||
// validate time (data-free: the [1, 2] 1970-epoch-ms window is never
|
||||
// reached), exit 1, with the executor fault header + the shape prose.
|
||||
let bad_proc = register_process_doc(dir, "wfmc.process.json", WF_AFTER_MC_PROCESS_DOC);
|
||||
write_doc(dir, "wfmc.campaign.json", &campaign_doc_json(&bp_id, &bad_proc, (1, 2), "", ""));
|
||||
let (bad_out, bad_code) = run_code_in(dir, &["campaign", "validate", "wfmc.campaign.json"]);
|
||||
assert_eq!(bad_code, Some(1), "an executor-unrunnable shape refuses at validate: {bad_out}");
|
||||
assert!(
|
||||
bad_out.contains("campaign is not executable:"),
|
||||
"the executor-tier fault header (mirrors 'campaign references do not resolve:'): {bad_out}"
|
||||
);
|
||||
assert!(
|
||||
bad_out.contains("std::walk_forward cannot follow std::monte_carlo"),
|
||||
"the shape prose names the ordering violation via exec_fault_prose: {bad_out}"
|
||||
);
|
||||
assert!(!bad_out.contains("PipelineShape"), "Debug leak: {bad_out}");
|
||||
}
|
||||
|
||||
/// A v2 process doc with a static `std::monte_carlo resamples: 0` defect.
|
||||
const ZERO_RESAMPLES_MC_PROCESS_DOC: &str = r#"{
|
||||
"format_version": 1,
|
||||
|
||||
Reference in New Issue
Block a user