feat(aura-runner, aura-cli): reproduce returns its report; mismatch guard replaces silent DIVERGED
reproduce_family returns Result<ReproduceReport, RunnerError> — divergence is a result the embedding caller reads (all_identical()), never an error; the CLI renders the per-member lines and summary byte-identically and owns the exit (1 on divergence, no stderr line). The library's print block and the empty-message refusal are gone; exit_on_runner_error always prints (no empty producer remains). reproduce_family_in gains the identity/pip guard, firing before any store fetch: an instrument mismatch (incl. synthetic-for-real) and a broker/pip mismatch — forward-built label compare against the stored broker string (RunManifest carries no pip field; the mint's label constructors became shared pub(crate) helpers, so guard and stamp cannot drift). The class is context-borne: the explicit-source seam refuses 2 (caller-named source), the derived simple path refuses 1 (data drift), same prose. A window guard is deliberately absent — no reproduce path takes a caller window. In-process tests pin the promise: divergence returns Ok, drift refuses class 1, the host survives. Fork minutes: issues/299#issuecomment-4875, -4877, -4879. closes #299
This commit is contained in:
+161
-12
@@ -1069,12 +1069,11 @@ pub(crate) fn exit_on_campaign_result(r: Result<usize, String>) {
|
||||
}
|
||||
}
|
||||
|
||||
/// The dispatch_reproduce pattern, shared: print the runner refusal and
|
||||
/// exit with its class (#297). Empty message = no stderr line.
|
||||
/// The shared runner-refusal arm: print the message and exit with its
|
||||
/// class (#297). Every producer carries prose since #299 retired the
|
||||
/// empty-message divergence refusal (divergence is report data now).
|
||||
fn exit_on_runner_error(e: aura_runner::RunnerError) -> ! {
|
||||
if !e.message.is_empty() {
|
||||
eprintln!("aura: {}", e.message);
|
||||
}
|
||||
eprintln!("aura: {}", e.message);
|
||||
std::process::exit(e.exit_code);
|
||||
}
|
||||
|
||||
@@ -1467,13 +1466,24 @@ fn dispatch_runs(a: RunsCmd, env: &aura_runner::project::Env) {
|
||||
}
|
||||
|
||||
fn dispatch_reproduce(a: ReproduceCmd, env: &aura_runner::project::Env) {
|
||||
// `reproduce_family` (#295) returns a `RunnerError` instead of exiting
|
||||
// the process itself; the shared helper prints its message (empty for
|
||||
// the "not every member reproduced" refusal, since no stderr line
|
||||
// accompanies it — only the stdout summary `reproduce_family` already
|
||||
// emitted) and exits with its code.
|
||||
if let Err(e) = aura_runner::reproduce::reproduce_family(&a.id, env) {
|
||||
exit_on_runner_error(e);
|
||||
// `reproduce_family` (#299) returns the report as a value — the library
|
||||
// never prints and never exits. This arm renders it (byte-identical to
|
||||
// the pre-#299 shell) and owns the exit: `exit(1)` when not every member
|
||||
// reproduced, with NO accompanying stderr line (unchanged observable).
|
||||
match aura_runner::reproduce::reproduce_family(&a.id, env) {
|
||||
Ok(report) => {
|
||||
let total = report.outcomes.len();
|
||||
let ok = report.outcomes.iter().filter(|(_, b)| *b).count();
|
||||
for (label, identical) in &report.outcomes {
|
||||
let verdict = if *identical { "bit-identical" } else { "DIVERGED" };
|
||||
println!("{} member {label} reproduced: {verdict}", a.id);
|
||||
}
|
||||
println!("reproduced {ok}/{total} members bit-identically");
|
||||
if ok != total {
|
||||
std::process::exit(1); // no stderr line — unchanged observable
|
||||
}
|
||||
}
|
||||
Err(e) => exit_on_runner_error(e),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1933,6 +1943,24 @@ mod tests {
|
||||
.expect("loads")
|
||||
}
|
||||
|
||||
/// An `Env` whose registry resolves to `dir` (#299): the
|
||||
/// project-facing surface `reproduce_family` reads via `env.registry()`,
|
||||
/// built directly (no `Aura.toml` on disk) by pointing `paths.runs` at the
|
||||
/// same absolute directory a test's own `Registry::open` uses — so seeding
|
||||
/// through the raw `Registry` handle and reading back through
|
||||
/// `reproduce_family`'s derived path see the identical `runs.jsonl`.
|
||||
fn env_over(dir: &std::path::Path) -> aura_runner::project::Env {
|
||||
aura_runner::project::Env::with_project(aura_runner::project::ProjectEnv {
|
||||
root: std::path::PathBuf::from("."),
|
||||
toml: aura_runner::project::AuraToml {
|
||||
paths: aura_runner::project::AuraPaths { data: None, runs: Some(dir.to_path_buf()) },
|
||||
nodes: Default::default(),
|
||||
},
|
||||
commit: None,
|
||||
native: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn cmp_member(key: &str, ts: &[i64], vals: &[f64]) -> FamilyMember {
|
||||
cmp_member_win(key, ts, vals, (0, 0))
|
||||
}
|
||||
@@ -2302,6 +2330,127 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// #299: `reproduce_family` (the derived-source seam behind
|
||||
/// plain `aura reproduce <id>`) survives a family whose STORED metrics were
|
||||
/// hand-falsified after minting — it returns `Ok(report)` with
|
||||
/// `all_identical() == false`, never a process exit or panic. The process
|
||||
/// obviously survives: this test returns normally and inspects the value
|
||||
/// `reproduce_family` handed back, not a captured exit code.
|
||||
#[test]
|
||||
fn reproduce_family_survives_a_falsified_stored_metric() {
|
||||
let dir = std::path::Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/../../target/tmp"))
|
||||
.join("aura-repro-falsified");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).expect("temp dir");
|
||||
let reg = Registry::open(dir.join("runs.jsonl"));
|
||||
|
||||
let env = aura_runner::project::Env::std();
|
||||
let closed = load_closed_r_sma();
|
||||
let doc = blueprint_to_json(&closed).expect("serializes");
|
||||
let reload = || blueprint_from_json(&doc, &|t| std_vocabulary(t)).expect("loads");
|
||||
let topo = test_topology_hash(&reload());
|
||||
let space = blueprint_axis_probe(&doc, &env).param_space();
|
||||
let data = DataSource::Synthetic;
|
||||
let pip = data.pip_size();
|
||||
let window = data.full_window(&env).expect("synthetic full_window never refuses");
|
||||
let binding = aura_runner::binding::resolve_binding("falsified", reload().input_roles(), &BTreeMap::new())
|
||||
.expect("the price role resolves");
|
||||
|
||||
let mut report = run_blueprint_member(
|
||||
reload(),
|
||||
&[],
|
||||
&space,
|
||||
data.run_sources(&env, &binding.columns()).expect("synthetic run_sources never refuses"),
|
||||
window,
|
||||
0,
|
||||
pip,
|
||||
&topo,
|
||||
&env,
|
||||
StopRule::Vol { length: R_SMA_STOP_LENGTH, k: R_SMA_STOP_K },
|
||||
&binding,
|
||||
&[],
|
||||
"",
|
||||
)
|
||||
.expect("no cost model: nothing to refuse");
|
||||
// Hand-falsify the STORED metric after an honest mint: the record on
|
||||
// disk now diverges from what a re-run actually produces — exactly the
|
||||
// property `reproduce_family` must surface as data, not a process exit.
|
||||
report.metrics.total_pips += 1.0;
|
||||
|
||||
let canonical = blueprint_to_json(&reload()).unwrap();
|
||||
reg.put_blueprint(&topo, &canonical).expect("store blueprint");
|
||||
let id = reg.append_family("falsified", FamilyKind::Sweep, &[report]).expect("append");
|
||||
|
||||
let rep = aura_runner::reproduce::reproduce_family(&id, &env_over(&dir))
|
||||
.expect("a divergent member is a reported outcome, not a refusal");
|
||||
assert!(!rep.all_identical(), "the falsified member must report DIVERGED: {:?}", rep.outcomes);
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// #299: the derived-source seam's identity/pip guard
|
||||
/// (Fork 6) refuses class 1, NOT class 2, on `reproduce_family` — a
|
||||
/// drifted stored broker label is data drift within the family (this
|
||||
/// path's `data` is reconstructed FROM the manifest, never caller-given),
|
||||
/// distinct from `reproduce_family_in`'s explicit-source class 2 (pinned
|
||||
/// in `aura_runner::reproduce`'s own test module).
|
||||
#[test]
|
||||
fn reproduce_family_reports_the_derived_path_mismatch_as_class_one() {
|
||||
let dir = std::path::Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/../../target/tmp"))
|
||||
.join("aura-repro-derived-class");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).expect("temp dir");
|
||||
let reg = Registry::open(dir.join("runs.jsonl"));
|
||||
|
||||
let env = aura_runner::project::Env::std();
|
||||
let closed = load_closed_r_sma();
|
||||
let doc = blueprint_to_json(&closed).expect("serializes");
|
||||
let reload = || blueprint_from_json(&doc, &|t| std_vocabulary(t)).expect("loads");
|
||||
let topo = test_topology_hash(&reload());
|
||||
let space = blueprint_axis_probe(&doc, &env).param_space();
|
||||
let data = DataSource::Synthetic;
|
||||
let pip = data.pip_size();
|
||||
let window = data.full_window(&env).expect("synthetic full_window never refuses");
|
||||
let binding = aura_runner::binding::resolve_binding("driftclass", reload().input_roles(), &BTreeMap::new())
|
||||
.expect("the price role resolves");
|
||||
|
||||
let mut report = run_blueprint_member(
|
||||
reload(),
|
||||
&[],
|
||||
&space,
|
||||
data.run_sources(&env, &binding.columns()).expect("synthetic run_sources never refuses"),
|
||||
window,
|
||||
0,
|
||||
pip,
|
||||
&topo,
|
||||
&env,
|
||||
StopRule::Vol { length: R_SMA_STOP_LENGTH, k: R_SMA_STOP_K },
|
||||
&binding,
|
||||
&[],
|
||||
"",
|
||||
)
|
||||
.expect("no cost model: nothing to refuse");
|
||||
// Drift the STORED broker label away from what this pip re-derives
|
||||
// (mirrors `aura_runner::reproduce`'s own pip-mismatch guard fixture,
|
||||
// but read back through the derived `reproduce_family` seam here).
|
||||
report.manifest.broker = "sim-optimal(pip_size=0.0002)".to_string();
|
||||
|
||||
let canonical = blueprint_to_json(&reload()).unwrap();
|
||||
reg.put_blueprint(&topo, &canonical).expect("store blueprint");
|
||||
let id = reg.append_family("driftclass", FamilyKind::Sweep, &[report]).expect("append");
|
||||
|
||||
let err = aura_runner::reproduce::reproduce_family(&id, &env_over(&dir))
|
||||
.expect_err("a drifted stored broker label must refuse, not silently reproduce");
|
||||
assert_eq!(err.exit_code, 1, "derived-path mismatch is class 1, not class 2: {}", err.message);
|
||||
assert!(
|
||||
err.message.contains("reproduce broker/pip mismatch"),
|
||||
"the refusal names the mismatch: {}",
|
||||
err.message
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reproduce_re_derives_the_member_stop_regime_not_the_default() {
|
||||
// Property: reproduce re-runs each member under the SAME stop regime the member
|
||||
|
||||
Reference in New Issue
Block a user