fix(runner, campaign): contain synthetic member panics like the real path (GREEN)
The #272 containment recipe (SilencedPanic + catch_unwind + panic_message) is exposed from aura-campaign as catch_member_panic and applied at the bare-member seam in the synthetic family builders: blueprint_sweep_family contains per-point, resolves the lowest enumeration index's fault after the sweep joins (thread-order- independent, C1), and exits deliberately with the aura: warning: class marker + exit 3 (C14) instead of the raw panic/exit-101 escape; blueprint_sweep_over captures instead of exiting (it runs inside the per-window walkforward parallel closure — an on-the-spot exit would race across windows) and hands the resolved fault up. Fault prose reuses member_fault_prose; the real path's containment is untouched. Deliberate residue, both tracked: blueprint_mc_family's member call site keeps the bare run (empirical probe of the reachable surface follows this commit; a reachable escape goes the same RED-first way), and the new eprintln+exit(3) site in aura-runner joins the #297 process::exit inventory (noted there) rather than fighting M4's future RunnerError propagation now. Both RED e2es green; workspace suite green via the mini-verify; clippy -D warnings clean on the touched crates. refs #278
This commit is contained in:
@@ -830,6 +830,19 @@ fn panic_message(payload: Box<dyn std::any::Any + Send>) -> String {
|
|||||||
.unwrap_or_else(|| "member panicked (non-string payload)".to_string())
|
.unwrap_or_else(|| "member panicked (non-string payload)".to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The `SilencedPanic` + `catch_unwind` + `panic_message` recipe above
|
||||||
|
/// (this module's three call sites), exposed for reuse by a member-run caller
|
||||||
|
/// OUTSIDE this crate that drives a bare member run without going through
|
||||||
|
/// `execute` (#278: the synthetic blueprint sweep/walk-forward family
|
||||||
|
/// builders in `aura-runner`, which share this exact bare-member seam but
|
||||||
|
/// carry no #272 fault boundary of their own). Returns the panic payload's
|
||||||
|
/// best-effort message on containment; `AssertUnwindSafe` is the same
|
||||||
|
/// judgement documented on `panic_message` above.
|
||||||
|
pub fn catch_member_panic<R>(f: impl FnOnce() -> R) -> Result<R, String> {
|
||||||
|
let _silence = SilencedPanic::enter();
|
||||||
|
std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).map_err(panic_message)
|
||||||
|
}
|
||||||
|
|
||||||
/// Assemble the (CellOutcome, CellRealization) pair for a cell that failed at a
|
/// Assemble the (CellOutcome, CellRealization) pair for a cell that failed at a
|
||||||
/// stage (#272): the realized stage prefix survives in `stages`, `families`/
|
/// stage (#272): the realized stage prefix survives in `stages`, `families`/
|
||||||
/// `selections` carry whatever already-persisted stages produced (empty for a
|
/// `selections` carry whatever already-persisted stages produced (empty for a
|
||||||
|
|||||||
@@ -13,7 +13,8 @@
|
|||||||
|
|
||||||
mod exec;
|
mod exec;
|
||||||
pub use exec::{
|
pub use exec::{
|
||||||
execute, member_fault_prose, CampaignOutcome, CellOutcome, StageFamily, StageSelectionOut,
|
catch_member_panic, execute, member_fault_prose, CampaignOutcome, CellOutcome, StageFamily,
|
||||||
|
StageSelectionOut,
|
||||||
};
|
};
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|||||||
@@ -15,8 +15,9 @@
|
|||||||
//! CLI shell still needs its own for call sites outside any family builder).
|
//! CLI shell still needs its own for call sites outside any family builder).
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::sync::Arc;
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use aura_campaign::{catch_member_panic, member_fault_prose, MemberFault};
|
||||||
use aura_composites::StopRule;
|
use aura_composites::StopRule;
|
||||||
use aura_core::{Cell, ParamSpec, Scalar, Timestamp};
|
use aura_core::{Cell, ParamSpec, Scalar, Timestamp};
|
||||||
use aura_engine::{
|
use aura_engine::{
|
||||||
@@ -315,7 +316,10 @@ pub fn select_winner(
|
|||||||
/// probe. `SweepBinder::sweep_with_lattice`'s own `resolve_axes`/arity/kind checks all
|
/// probe. `SweepBinder::sweep_with_lattice`'s own `resolve_axes`/arity/kind checks all
|
||||||
/// run BEFORE this closure is invoked per grid point, so its body never influences the
|
/// run BEFORE this closure is invoked per grid point, so its body never influences the
|
||||||
/// validation outcome — only its signature (`Fn(&[Cell]) -> RunReport`) needs to
|
/// validation outcome — only its signature (`Fn(&[Cell]) -> RunReport`) needs to
|
||||||
/// satisfy the terminal, at O(1) cost per point instead of a full member run.
|
/// satisfy the terminal, at O(1) cost per point instead of a full member run. Reused
|
||||||
|
/// (#278) as the discarded slot value for a member run [`catch_member_panic`]
|
||||||
|
/// contains: the caller always resolves the captured fault and exits before this
|
||||||
|
/// value is ever rendered or persisted, so its content is equally irrelevant there.
|
||||||
fn axis_grid_probe_report() -> RunReport {
|
fn axis_grid_probe_report() -> RunReport {
|
||||||
RunReport {
|
RunReport {
|
||||||
manifest: RunManifest {
|
manifest: RunManifest {
|
||||||
@@ -334,6 +338,34 @@ fn axis_grid_probe_report() -> RunReport {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The LOWEST enumeration index's captured member-panic message, found by
|
||||||
|
/// walking `family.points` — guaranteed by `assemble_sweep`'s own
|
||||||
|
/// `collect()` to be in grid-enumeration order regardless of which worker
|
||||||
|
/// finished first (C1) — for the first point whose params match a captured
|
||||||
|
/// fault. Thread-order-independent by construction (mirrors the lowest-
|
||||||
|
/// index-fault convention `aura-campaign::exec::run_members` establishes for
|
||||||
|
/// the identical #272 seam), without needing a separate numeric-index
|
||||||
|
/// side-channel: the returned family already carries the points in order.
|
||||||
|
fn lowest_point_fault(family: &SweepFamily, faults: Mutex<Vec<(Vec<Cell>, String)>>) -> Option<String> {
|
||||||
|
let captured = faults.into_inner().expect("fault capture lock");
|
||||||
|
family
|
||||||
|
.points
|
||||||
|
.iter()
|
||||||
|
.find_map(|pt| captured.iter().find(|(p, _)| *p == pt.params).map(|(_, m)| m.clone()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Render + exit(3) on a contained member panic (#278: the deliberate
|
||||||
|
/// failed-cells exit, C14, never the raw 101 an uncontained panic would
|
||||||
|
/// yield) — reuses `aura_campaign::member_fault_prose`'s established
|
||||||
|
/// "a member panicked: …" wording rather than re-wording it, and the
|
||||||
|
/// `aura: warning: ` class marker (#278 decision) the CLI's own `diag`
|
||||||
|
/// macro would emit, hand-written here since that macro is presentation-
|
||||||
|
/// crate-private (#295 boundary) and this fires from `aura-runner`.
|
||||||
|
fn exit_on_member_panic(msg: &str) -> ! {
|
||||||
|
eprintln!("aura: warning: {}", member_fault_prose(&MemberFault::Panic(msg.to_string())));
|
||||||
|
std::process::exit(3);
|
||||||
|
}
|
||||||
|
|
||||||
/// Validate the `--axis` grid against `doc`'s wrapped param space WITHOUT running any
|
/// Validate the `--axis` grid against `doc`'s wrapped param space WITHOUT running any
|
||||||
/// member (#253). Reuses the SAME strict, erroring axis-name check
|
/// member (#253). Reuses the SAME strict, erroring axis-name check
|
||||||
/// `blueprint_sweep_over` performs (`override_paths` — single-sourced, so the two
|
/// `blueprint_sweep_over` performs (`override_paths` — single-sourced, so the two
|
||||||
@@ -426,15 +458,35 @@ pub fn blueprint_sweep_family(
|
|||||||
for (n, vals) in iter {
|
for (n, vals) in iter {
|
||||||
binder = binder.axis(n, vals.clone());
|
binder = binder.axis(n, vals.clone());
|
||||||
}
|
}
|
||||||
binder
|
// #278: `run_blueprint_member` runs bare here (no #272 fault boundary of its
|
||||||
|
// own), so a member-compile panic (e.g. an `Sma::new` length assert) would
|
||||||
|
// otherwise unwind straight through this sweep to an uncaught exit 101 —
|
||||||
|
// contained the same way the real/campaign path contains it, via
|
||||||
|
// `catch_member_panic` + a per-point capture, resolved to the lowest
|
||||||
|
// enumeration index's message after the sweep joins (thread-order-
|
||||||
|
// independent, C1).
|
||||||
|
let faults: Mutex<Vec<(Vec<Cell>, String)>> = Mutex::new(Vec::new());
|
||||||
|
let family = binder
|
||||||
.sweep(|point| {
|
.sweep(|point| {
|
||||||
// fresh per-member graph (Composite is !Clone, reload per member) run through
|
// fresh per-member graph (Composite is !Clone, reload per member) run through
|
||||||
// the shared reduce-mode member path — the same fn reproduction re-runs.
|
// the shared reduce-mode member path — the same fn reproduction re-runs.
|
||||||
run_blueprint_member(reload(doc), point, &space, data.run_sources(env, &binding.columns()), window, 0, pip, &topo, env, DEFAULT_STOP, &binding, &[], NO_INSTRUMENT_CONTEXT)
|
match catch_member_panic(|| {
|
||||||
|
run_blueprint_member(reload(doc), point, &space, data.run_sources(env, &binding.columns()), window, 0, pip, &topo, env, DEFAULT_STOP, &binding, &[], NO_INSTRUMENT_CONTEXT)
|
||||||
|
}) {
|
||||||
|
Ok(report) => report,
|
||||||
|
Err(msg) => {
|
||||||
|
faults.lock().expect("fault capture lock").push((point.to_vec(), msg));
|
||||||
|
axis_grid_probe_report()
|
||||||
|
}
|
||||||
|
}
|
||||||
})
|
})
|
||||||
// render the sweep terminal's BindError to prose (#247), the fn's String error
|
// render the sweep terminal's BindError to prose (#247), the fn's String error
|
||||||
// contract — never the raw Debug struct.
|
// contract — never the raw Debug struct.
|
||||||
.map_err(|e| render_bind_error(&e))
|
.map_err(|e| render_bind_error(&e))?;
|
||||||
|
if let Some(msg) = lowest_point_fault(&family, faults) {
|
||||||
|
exit_on_member_panic(&msg);
|
||||||
|
}
|
||||||
|
Ok(family)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Sweep the LOADED blueprint over the user `--axis` grid on an in-sample window
|
/// Sweep the LOADED blueprint over the user `--axis` grid on an in-sample window
|
||||||
@@ -447,10 +499,19 @@ pub fn blueprint_sweep_family(
|
|||||||
/// its walk-forward in-sample twin); an axis matching neither space is refused
|
/// its walk-forward in-sample twin); an axis matching neither space is refused
|
||||||
/// (wrapped as `BindError::UnknownKnob`, the honest replacement for the retired
|
/// (wrapped as `BindError::UnknownKnob`, the honest replacement for the retired
|
||||||
/// "fully bound; nothing to sweep" refusal) before any member runs.
|
/// "fully bound; nothing to sweep" refusal) before any member runs.
|
||||||
|
///
|
||||||
|
/// The returned `Option<String>` (#278) is this fn's OWN resolved member-panic
|
||||||
|
/// capture: this fn's nested sweep runs inside `blueprint_walkforward_family`'s
|
||||||
|
/// per-window parallel closure, so a contained member panic here must NOT
|
||||||
|
/// `eprintln!`+`exit` on the spot (that would race across windows, the exact
|
||||||
|
/// `#177` duplicated-rejection class the dispatch-boundary axis pre-flight
|
||||||
|
/// above already avoids) — captured internally, resolved to the lowest-index
|
||||||
|
/// message via [`lowest_point_fault`] before returning, for the caller to push
|
||||||
|
/// onto its own per-window fault list only after every window has joined.
|
||||||
pub fn blueprint_sweep_over(
|
pub fn blueprint_sweep_over(
|
||||||
doc: &str, axes: &[(String, Vec<Scalar>)], from: Timestamp, to: Timestamp, data: &DataSource,
|
doc: &str, axes: &[(String, Vec<Scalar>)], from: Timestamp, to: Timestamp, data: &DataSource,
|
||||||
env: &Env, binding: &ResolvedBinding,
|
env: &Env, binding: &ResolvedBinding,
|
||||||
) -> Result<(SweepFamily, Vec<usize>), BindError> {
|
) -> Result<(SweepFamily, Vec<usize>, Option<String>), BindError> {
|
||||||
let reload = |d: &str| {
|
let reload = |d: &str| {
|
||||||
blueprint_from_json(d, &|t| env.resolve(t))
|
blueprint_from_json(d, &|t| env.resolve(t))
|
||||||
.expect("doc parse-validated at the dispatch boundary; reload is infallible")
|
.expect("doc parse-validated at the dispatch boundary; reload is infallible")
|
||||||
@@ -476,11 +537,22 @@ pub fn blueprint_sweep_over(
|
|||||||
for (n, vals) in iter {
|
for (n, vals) in iter {
|
||||||
binder = binder.axis(n, vals.clone());
|
binder = binder.axis(n, vals.clone());
|
||||||
}
|
}
|
||||||
binder.sweep_with_lattice(|point| {
|
let faults: Mutex<Vec<(Vec<Cell>, String)>> = Mutex::new(Vec::new());
|
||||||
|
let (family, lattice) = binder.sweep_with_lattice(|point| {
|
||||||
let sources = data.windowed_sources(from, to, env, &binding.columns());
|
let sources = data.windowed_sources(from, to, env, &binding.columns());
|
||||||
let window = window_of(&sources).expect("non-empty in-sample window");
|
let window = window_of(&sources).expect("non-empty in-sample window");
|
||||||
run_blueprint_member(reopen_all(reload(doc), &overrides), point, &space, sources, window, 0, pip, &topo, env, DEFAULT_STOP, binding, &[], NO_INSTRUMENT_CONTEXT)
|
match catch_member_panic(|| {
|
||||||
})
|
run_blueprint_member(reopen_all(reload(doc), &overrides), point, &space, sources, window, 0, pip, &topo, env, DEFAULT_STOP, binding, &[], NO_INSTRUMENT_CONTEXT)
|
||||||
|
}) {
|
||||||
|
Ok(report) => report,
|
||||||
|
Err(msg) => {
|
||||||
|
faults.lock().expect("fault capture lock").push((point.to_vec(), msg));
|
||||||
|
axis_grid_probe_report()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
let fault = lowest_point_fault(&family, faults);
|
||||||
|
Ok((family, lattice, fault))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run the winner params over an out-of-sample window `[from,to]` on the loaded
|
/// Run the winner params over an out-of-sample window `[from,to]` on the loaded
|
||||||
@@ -509,6 +581,37 @@ pub fn run_oos_blueprint(
|
|||||||
(Vec::new(), report)
|
(Vec::new(), report)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A discarded placeholder [`WindowRun`] for a window whose IS refit or OOS
|
||||||
|
/// run was contained after a member panic (#278): `chosen_params` matches
|
||||||
|
/// `space`'s arity with inert zero cells (the `assemble_walk_forward` arity
|
||||||
|
/// invariant), `oos_equity` is empty (an empty segment leaves the stitched
|
||||||
|
/// curve unbroken, same convention as a genuinely equity-less window), and
|
||||||
|
/// the report reuses [`axis_grid_probe_report`]'s zero-cost placeholder.
|
||||||
|
/// Never rendered: the caller always resolves the captured window fault and
|
||||||
|
/// exits before touching a faulted window's run.
|
||||||
|
fn placeholder_window_run(space: &[ParamSpec]) -> WindowRun {
|
||||||
|
WindowRun {
|
||||||
|
chosen_params: space.iter().map(|_| Cell::from_i64(0)).collect(),
|
||||||
|
oos_equity: Vec::new(),
|
||||||
|
oos_report: axis_grid_probe_report(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The LOWEST roll-order window's captured member-panic message (#278),
|
||||||
|
/// found by walking `result.windows` — guaranteed by `assemble_walk_forward`
|
||||||
|
/// to be in roll order regardless of thread completion (C1) — for the first
|
||||||
|
/// window whose bounds match a captured fault. Thread-order-independent by
|
||||||
|
/// construction, mirroring [`lowest_point_fault`]'s identical convention one
|
||||||
|
/// level down (per-point within one window's IS sweep) and
|
||||||
|
/// `aura-campaign::exec`'s own lowest-index walk-forward fault attribution.
|
||||||
|
fn lowest_window_fault(result: &WalkForwardResult, faults: Mutex<Vec<(WindowBounds, String)>>) -> Option<String> {
|
||||||
|
let captured = faults.into_inner().expect("fault capture lock");
|
||||||
|
result
|
||||||
|
.windows
|
||||||
|
.iter()
|
||||||
|
.find_map(|w| captured.iter().find(|(b, _)| *b == w.bounds).map(|(_, m)| m.clone()))
|
||||||
|
}
|
||||||
|
|
||||||
/// The loaded-blueprint IS-refit walk-forward: per IS window, re-optimize the
|
/// The loaded-blueprint IS-refit walk-forward: per IS window, re-optimize the
|
||||||
/// blueprint over the user `--axis` grid, select by `sqn_normalized`, run the
|
/// blueprint over the user `--axis` grid, select by `sqn_normalized`, run the
|
||||||
/// winner OOS, reusing the generic `walk_forward` driver + `select_winner`;
|
/// winner OOS, reusing the generic `walk_forward` driver + `select_winner`;
|
||||||
@@ -568,18 +671,45 @@ pub fn blueprint_walkforward_family(
|
|||||||
eprintln!("aura: {}", render_bind_error(&e));
|
eprintln!("aura: {}", render_bind_error(&e));
|
||||||
std::process::exit(2);
|
std::process::exit(2);
|
||||||
}
|
}
|
||||||
walk_forward(roller, space.clone(), |w: WindowBounds| {
|
// #278: `blueprint_sweep_over`'s inner sweep and `run_oos_blueprint`'s single
|
||||||
let (is_family, lattice) = blueprint_sweep_over(doc, axes, w.is.0, w.is.1, data, env, &binding)
|
// member run both drive `run_blueprint_member` bare, so either could
|
||||||
|
// otherwise unwind an uncaught member-compile panic straight through this
|
||||||
|
// window closure. Both are contained (`blueprint_sweep_over` resolves its
|
||||||
|
// own capture and returns it; `run_oos_blueprint` via `catch_member_panic`
|
||||||
|
// directly) and captured PER WINDOW here rather than printed on the spot —
|
||||||
|
// this closure runs across windows in parallel (`walk_forward`), so an
|
||||||
|
// inline `eprintln!`+`exit` would race the same way the axis pre-flight
|
||||||
|
// above was written to avoid (#177). The lowest roll-order window's fault
|
||||||
|
// is resolved once, after every window has joined.
|
||||||
|
let window_faults: Mutex<Vec<(WindowBounds, String)>> = Mutex::new(Vec::new());
|
||||||
|
let result = walk_forward(roller, space.clone(), |w: WindowBounds| {
|
||||||
|
let (is_family, lattice, is_fault) = blueprint_sweep_over(doc, axes, w.is.0, w.is.1, data, env, &binding)
|
||||||
.expect("axes validated in the dispatch-boundary pre-flight");
|
.expect("axes validated in the dispatch-boundary pre-flight");
|
||||||
|
if let Some(msg) = is_fault {
|
||||||
|
window_faults.lock().expect("fault capture lock").push((w, msg));
|
||||||
|
return placeholder_window_run(&space);
|
||||||
|
}
|
||||||
let (best, selection) = match select_winner(&is_family, WINNER_SELECTION_METRIC, select, Some(&lattice)) {
|
let (best, selection) = match select_winner(&is_family, WINNER_SELECTION_METRIC, select, Some(&lattice)) {
|
||||||
Ok(v) => v,
|
Ok(v) => v,
|
||||||
Err(msg) => { eprintln!("aura: {msg}"); std::process::exit(2); }
|
Err(msg) => { eprintln!("aura: {msg}"); std::process::exit(2); }
|
||||||
};
|
};
|
||||||
let (oos_equity, mut oos_report) =
|
match catch_member_panic(|| {
|
||||||
run_oos_blueprint(doc, &best.params, &space, w.oos.0, w.oos.1, &topo, data, env, &binding, &overrides);
|
run_oos_blueprint(doc, &best.params, &space, w.oos.0, w.oos.1, &topo, data, env, &binding, &overrides)
|
||||||
oos_report.manifest.selection = Some(selection);
|
}) {
|
||||||
WindowRun { chosen_params: best.params, oos_equity, oos_report }
|
Ok((oos_equity, mut oos_report)) => {
|
||||||
})
|
oos_report.manifest.selection = Some(selection);
|
||||||
|
WindowRun { chosen_params: best.params, oos_equity, oos_report }
|
||||||
|
}
|
||||||
|
Err(msg) => {
|
||||||
|
window_faults.lock().expect("fault capture lock").push((w, msg));
|
||||||
|
placeholder_window_run(&space)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
if let Some(msg) = lowest_window_fault(&result, window_faults) {
|
||||||
|
exit_on_member_panic(&msg);
|
||||||
|
}
|
||||||
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A fresh seeded synthetic price walk for one Monte-Carlo draw — `blueprint_mc_family`'s
|
/// A fresh seeded synthetic price walk for one Monte-Carlo draw — `blueprint_mc_family`'s
|
||||||
|
|||||||
Reference in New Issue
Block a user