fix(runner, campaign): contain synthetic member panics like the real path (GREEN)

The #272 containment recipe (SilencedPanic + catch_unwind +
panic_message) is exposed from aura-campaign as catch_member_panic and
applied at the bare-member seam in the synthetic family builders:
blueprint_sweep_family contains per-point, resolves the lowest
enumeration index's fault after the sweep joins (thread-order-
independent, C1), and exits deliberately with the aura: warning:
class marker + exit 3 (C14) instead of the raw panic/exit-101 escape;
blueprint_sweep_over captures instead of exiting (it runs inside the
per-window walkforward parallel closure — an on-the-spot exit would
race across windows) and hands the resolved fault up. Fault prose
reuses member_fault_prose; the real path's containment is untouched.

Deliberate residue, both tracked: blueprint_mc_family's member call
site keeps the bare run (empirical probe of the reachable surface
follows this commit; a reachable escape goes the same RED-first way),
and the new eprintln+exit(3) site in aura-runner joins the #297
process::exit inventory (noted there) rather than fighting M4's
future RunnerError propagation now.

Both RED e2es green; workspace suite green via the mini-verify;
clippy -D warnings clean on the touched crates. refs #278
This commit is contained in:
2026-07-24 00:43:06 +02:00
parent bd6ddf9efe
commit 51096a3212
3 changed files with 161 additions and 17 deletions
+13
View File
@@ -830,6 +830,19 @@ fn panic_message(payload: Box<dyn std::any::Any + Send>) -> String {
.unwrap_or_else(|| "member panicked (non-string payload)".to_string())
}
/// The `SilencedPanic` + `catch_unwind` + `panic_message` recipe above
/// (this module's three call sites), exposed for reuse by a member-run caller
/// OUTSIDE this crate that drives a bare member run without going through
/// `execute` (#278: the synthetic blueprint sweep/walk-forward family
/// builders in `aura-runner`, which share this exact bare-member seam but
/// carry no #272 fault boundary of their own). Returns the panic payload's
/// best-effort message on containment; `AssertUnwindSafe` is the same
/// judgement documented on `panic_message` above.
pub fn catch_member_panic<R>(f: impl FnOnce() -> R) -> Result<R, String> {
let _silence = SilencedPanic::enter();
std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).map_err(panic_message)
}
/// Assemble the (CellOutcome, CellRealization) pair for a cell that failed at a
/// stage (#272): the realized stage prefix survives in `stages`, `families`/
/// `selections` carry whatever already-persisted stages produced (empty for a
+2 -1
View File
@@ -13,7 +13,8 @@
mod exec;
pub use exec::{
execute, member_fault_prose, CampaignOutcome, CellOutcome, StageFamily, StageSelectionOut,
catch_member_panic, execute, member_fault_prose, CampaignOutcome, CellOutcome, StageFamily,
StageSelectionOut,
};
use std::collections::BTreeMap;
+142 -12
View File
@@ -15,8 +15,9 @@
//! CLI shell still needs its own for call sites outside any family builder).
use std::collections::BTreeMap;
use std::sync::Arc;
use std::sync::{Arc, Mutex};
use aura_campaign::{catch_member_panic, member_fault_prose, MemberFault};
use aura_composites::StopRule;
use aura_core::{Cell, ParamSpec, Scalar, Timestamp};
use aura_engine::{
@@ -315,7 +316,10 @@ pub fn select_winner(
/// probe. `SweepBinder::sweep_with_lattice`'s own `resolve_axes`/arity/kind checks all
/// run BEFORE this closure is invoked per grid point, so its body never influences the
/// validation outcome — only its signature (`Fn(&[Cell]) -> RunReport`) needs to
/// satisfy the terminal, at O(1) cost per point instead of a full member run.
/// satisfy the terminal, at O(1) cost per point instead of a full member run. Reused
/// (#278) as the discarded slot value for a member run [`catch_member_panic`]
/// contains: the caller always resolves the captured fault and exits before this
/// value is ever rendered or persisted, so its content is equally irrelevant there.
fn axis_grid_probe_report() -> RunReport {
RunReport {
manifest: RunManifest {
@@ -334,6 +338,34 @@ fn axis_grid_probe_report() -> RunReport {
}
}
/// The LOWEST enumeration index's captured member-panic message, found by
/// walking `family.points` — guaranteed by `assemble_sweep`'s own
/// `collect()` to be in grid-enumeration order regardless of which worker
/// finished first (C1) — for the first point whose params match a captured
/// fault. Thread-order-independent by construction (mirrors the lowest-
/// index-fault convention `aura-campaign::exec::run_members` establishes for
/// the identical #272 seam), without needing a separate numeric-index
/// side-channel: the returned family already carries the points in order.
fn lowest_point_fault(family: &SweepFamily, faults: Mutex<Vec<(Vec<Cell>, String)>>) -> Option<String> {
let captured = faults.into_inner().expect("fault capture lock");
family
.points
.iter()
.find_map(|pt| captured.iter().find(|(p, _)| *p == pt.params).map(|(_, m)| m.clone()))
}
/// Render + exit(3) on a contained member panic (#278: the deliberate
/// failed-cells exit, C14, never the raw 101 an uncontained panic would
/// yield) — reuses `aura_campaign::member_fault_prose`'s established
/// "a member panicked: …" wording rather than re-wording it, and the
/// `aura: warning: ` class marker (#278 decision) the CLI's own `diag`
/// macro would emit, hand-written here since that macro is presentation-
/// crate-private (#295 boundary) and this fires from `aura-runner`.
fn exit_on_member_panic(msg: &str) -> ! {
eprintln!("aura: warning: {}", member_fault_prose(&MemberFault::Panic(msg.to_string())));
std::process::exit(3);
}
/// Validate the `--axis` grid against `doc`'s wrapped param space WITHOUT running any
/// member (#253). Reuses the SAME strict, erroring axis-name check
/// `blueprint_sweep_over` performs (`override_paths` — single-sourced, so the two
@@ -426,15 +458,35 @@ pub fn blueprint_sweep_family(
for (n, vals) in iter {
binder = binder.axis(n, vals.clone());
}
binder
// #278: `run_blueprint_member` runs bare here (no #272 fault boundary of its
// own), so a member-compile panic (e.g. an `Sma::new` length assert) would
// otherwise unwind straight through this sweep to an uncaught exit 101 —
// contained the same way the real/campaign path contains it, via
// `catch_member_panic` + a per-point capture, resolved to the lowest
// enumeration index's message after the sweep joins (thread-order-
// independent, C1).
let faults: Mutex<Vec<(Vec<Cell>, String)>> = Mutex::new(Vec::new());
let family = binder
.sweep(|point| {
// fresh per-member graph (Composite is !Clone, reload per member) run through
// the shared reduce-mode member path — the same fn reproduction re-runs.
match catch_member_panic(|| {
run_blueprint_member(reload(doc), point, &space, data.run_sources(env, &binding.columns()), window, 0, pip, &topo, env, DEFAULT_STOP, &binding, &[], NO_INSTRUMENT_CONTEXT)
}) {
Ok(report) => report,
Err(msg) => {
faults.lock().expect("fault capture lock").push((point.to_vec(), msg));
axis_grid_probe_report()
}
}
})
// render the sweep terminal's BindError to prose (#247), the fn's String error
// contract — never the raw Debug struct.
.map_err(|e| render_bind_error(&e))
.map_err(|e| render_bind_error(&e))?;
if let Some(msg) = lowest_point_fault(&family, faults) {
exit_on_member_panic(&msg);
}
Ok(family)
}
/// Sweep the LOADED blueprint over the user `--axis` grid on an in-sample window
@@ -447,10 +499,19 @@ pub fn blueprint_sweep_family(
/// its walk-forward in-sample twin); an axis matching neither space is refused
/// (wrapped as `BindError::UnknownKnob`, the honest replacement for the retired
/// "fully bound; nothing to sweep" refusal) before any member runs.
///
/// The returned `Option<String>` (#278) is this fn's OWN resolved member-panic
/// capture: this fn's nested sweep runs inside `blueprint_walkforward_family`'s
/// per-window parallel closure, so a contained member panic here must NOT
/// `eprintln!`+`exit` on the spot (that would race across windows, the exact
/// `#177` duplicated-rejection class the dispatch-boundary axis pre-flight
/// above already avoids) — captured internally, resolved to the lowest-index
/// message via [`lowest_point_fault`] before returning, for the caller to push
/// onto its own per-window fault list only after every window has joined.
pub fn blueprint_sweep_over(
doc: &str, axes: &[(String, Vec<Scalar>)], from: Timestamp, to: Timestamp, data: &DataSource,
env: &Env, binding: &ResolvedBinding,
) -> Result<(SweepFamily, Vec<usize>), BindError> {
) -> Result<(SweepFamily, Vec<usize>, Option<String>), BindError> {
let reload = |d: &str| {
blueprint_from_json(d, &|t| env.resolve(t))
.expect("doc parse-validated at the dispatch boundary; reload is infallible")
@@ -476,11 +537,22 @@ pub fn blueprint_sweep_over(
for (n, vals) in iter {
binder = binder.axis(n, vals.clone());
}
binder.sweep_with_lattice(|point| {
let faults: Mutex<Vec<(Vec<Cell>, String)>> = Mutex::new(Vec::new());
let (family, lattice) = binder.sweep_with_lattice(|point| {
let sources = data.windowed_sources(from, to, env, &binding.columns());
let window = window_of(&sources).expect("non-empty in-sample window");
match catch_member_panic(|| {
run_blueprint_member(reopen_all(reload(doc), &overrides), point, &space, sources, window, 0, pip, &topo, env, DEFAULT_STOP, binding, &[], NO_INSTRUMENT_CONTEXT)
})
}) {
Ok(report) => report,
Err(msg) => {
faults.lock().expect("fault capture lock").push((point.to_vec(), msg));
axis_grid_probe_report()
}
}
})?;
let fault = lowest_point_fault(&family, faults);
Ok((family, lattice, fault))
}
/// Run the winner params over an out-of-sample window `[from,to]` on the loaded
@@ -509,6 +581,37 @@ pub fn run_oos_blueprint(
(Vec::new(), report)
}
/// A discarded placeholder [`WindowRun`] for a window whose IS refit or OOS
/// run was contained after a member panic (#278): `chosen_params` matches
/// `space`'s arity with inert zero cells (the `assemble_walk_forward` arity
/// invariant), `oos_equity` is empty (an empty segment leaves the stitched
/// curve unbroken, same convention as a genuinely equity-less window), and
/// the report reuses [`axis_grid_probe_report`]'s zero-cost placeholder.
/// Never rendered: the caller always resolves the captured window fault and
/// exits before touching a faulted window's run.
fn placeholder_window_run(space: &[ParamSpec]) -> WindowRun {
WindowRun {
chosen_params: space.iter().map(|_| Cell::from_i64(0)).collect(),
oos_equity: Vec::new(),
oos_report: axis_grid_probe_report(),
}
}
/// The LOWEST roll-order window's captured member-panic message (#278),
/// found by walking `result.windows` — guaranteed by `assemble_walk_forward`
/// to be in roll order regardless of thread completion (C1) — for the first
/// window whose bounds match a captured fault. Thread-order-independent by
/// construction, mirroring [`lowest_point_fault`]'s identical convention one
/// level down (per-point within one window's IS sweep) and
/// `aura-campaign::exec`'s own lowest-index walk-forward fault attribution.
fn lowest_window_fault(result: &WalkForwardResult, faults: Mutex<Vec<(WindowBounds, String)>>) -> Option<String> {
let captured = faults.into_inner().expect("fault capture lock");
result
.windows
.iter()
.find_map(|w| captured.iter().find(|(b, _)| *b == w.bounds).map(|(_, m)| m.clone()))
}
/// The loaded-blueprint IS-refit walk-forward: per IS window, re-optimize the
/// blueprint over the user `--axis` grid, select by `sqn_normalized`, run the
/// winner OOS, reusing the generic `walk_forward` driver + `select_winner`;
@@ -568,18 +671,45 @@ pub fn blueprint_walkforward_family(
eprintln!("aura: {}", render_bind_error(&e));
std::process::exit(2);
}
walk_forward(roller, space.clone(), |w: WindowBounds| {
let (is_family, lattice) = blueprint_sweep_over(doc, axes, w.is.0, w.is.1, data, env, &binding)
// #278: `blueprint_sweep_over`'s inner sweep and `run_oos_blueprint`'s single
// member run both drive `run_blueprint_member` bare, so either could
// otherwise unwind an uncaught member-compile panic straight through this
// window closure. Both are contained (`blueprint_sweep_over` resolves its
// own capture and returns it; `run_oos_blueprint` via `catch_member_panic`
// directly) and captured PER WINDOW here rather than printed on the spot —
// this closure runs across windows in parallel (`walk_forward`), so an
// inline `eprintln!`+`exit` would race the same way the axis pre-flight
// above was written to avoid (#177). The lowest roll-order window's fault
// is resolved once, after every window has joined.
let window_faults: Mutex<Vec<(WindowBounds, String)>> = Mutex::new(Vec::new());
let result = walk_forward(roller, space.clone(), |w: WindowBounds| {
let (is_family, lattice, is_fault) = blueprint_sweep_over(doc, axes, w.is.0, w.is.1, data, env, &binding)
.expect("axes validated in the dispatch-boundary pre-flight");
if let Some(msg) = is_fault {
window_faults.lock().expect("fault capture lock").push((w, msg));
return placeholder_window_run(&space);
}
let (best, selection) = match select_winner(&is_family, WINNER_SELECTION_METRIC, select, Some(&lattice)) {
Ok(v) => v,
Err(msg) => { eprintln!("aura: {msg}"); std::process::exit(2); }
};
let (oos_equity, mut oos_report) =
run_oos_blueprint(doc, &best.params, &space, w.oos.0, w.oos.1, &topo, data, env, &binding, &overrides);
match catch_member_panic(|| {
run_oos_blueprint(doc, &best.params, &space, w.oos.0, w.oos.1, &topo, data, env, &binding, &overrides)
}) {
Ok((oos_equity, mut oos_report)) => {
oos_report.manifest.selection = Some(selection);
WindowRun { chosen_params: best.params, oos_equity, oos_report }
})
}
Err(msg) => {
window_faults.lock().expect("fault capture lock").push((w, msg));
placeholder_window_run(&space)
}
}
});
if let Some(msg) = lowest_window_fault(&result, window_faults) {
exit_on_member_panic(&msg);
}
result
}
/// A fresh seeded synthetic price walk for one Monte-Carlo draw — `blueprint_mc_family`'s