diff --git a/crates/aura-campaign/src/exec.rs b/crates/aura-campaign/src/exec.rs index f068fbb..bbc9f57 100644 --- a/crates/aura-campaign/src/exec.rs +++ b/crates/aura-campaign/src/exec.rs @@ -830,6 +830,19 @@ fn panic_message(payload: Box) -> String { .unwrap_or_else(|| "member panicked (non-string payload)".to_string()) } +/// The `SilencedPanic` + `catch_unwind` + `panic_message` recipe above +/// (this module's three call sites), exposed for reuse by a member-run caller +/// OUTSIDE this crate that drives a bare member run without going through +/// `execute` (#278: the synthetic blueprint sweep/walk-forward family +/// builders in `aura-runner`, which share this exact bare-member seam but +/// carry no #272 fault boundary of their own). Returns the panic payload's +/// best-effort message on containment; `AssertUnwindSafe` is the same +/// judgement documented on `panic_message` above. +pub fn catch_member_panic(f: impl FnOnce() -> R) -> Result { + let _silence = SilencedPanic::enter(); + std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).map_err(panic_message) +} + /// Assemble the (CellOutcome, CellRealization) pair for a cell that failed at a /// stage (#272): the realized stage prefix survives in `stages`, `families`/ /// `selections` carry whatever already-persisted stages produced (empty for a diff --git a/crates/aura-campaign/src/lib.rs b/crates/aura-campaign/src/lib.rs index 3fbb3cf..a4ed4ea 100644 --- a/crates/aura-campaign/src/lib.rs +++ b/crates/aura-campaign/src/lib.rs @@ -13,7 +13,8 @@ mod exec; pub use exec::{ - execute, member_fault_prose, CampaignOutcome, CellOutcome, StageFamily, StageSelectionOut, + catch_member_panic, execute, member_fault_prose, CampaignOutcome, CellOutcome, StageFamily, + StageSelectionOut, }; use std::collections::BTreeMap; diff --git a/crates/aura-runner/src/family.rs b/crates/aura-runner/src/family.rs index c53b2bc..d77a823 100644 --- a/crates/aura-runner/src/family.rs +++ b/crates/aura-runner/src/family.rs @@ -15,8 +15,9 @@ //! CLI shell still needs its own for call sites outside any family builder). use std::collections::BTreeMap; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; +use aura_campaign::{catch_member_panic, member_fault_prose, MemberFault}; use aura_composites::StopRule; use aura_core::{Cell, ParamSpec, Scalar, Timestamp}; use aura_engine::{ @@ -315,7 +316,10 @@ pub fn select_winner( /// probe. `SweepBinder::sweep_with_lattice`'s own `resolve_axes`/arity/kind checks all /// run BEFORE this closure is invoked per grid point, so its body never influences the /// validation outcome — only its signature (`Fn(&[Cell]) -> RunReport`) needs to -/// satisfy the terminal, at O(1) cost per point instead of a full member run. +/// satisfy the terminal, at O(1) cost per point instead of a full member run. Reused +/// (#278) as the discarded slot value for a member run [`catch_member_panic`] +/// contains: the caller always resolves the captured fault and exits before this +/// value is ever rendered or persisted, so its content is equally irrelevant there. fn axis_grid_probe_report() -> RunReport { RunReport { manifest: RunManifest { @@ -334,6 +338,34 @@ fn axis_grid_probe_report() -> RunReport { } } +/// The LOWEST enumeration index's captured member-panic message, found by +/// walking `family.points` — guaranteed by `assemble_sweep`'s own +/// `collect()` to be in grid-enumeration order regardless of which worker +/// finished first (C1) — for the first point whose params match a captured +/// fault. Thread-order-independent by construction (mirrors the lowest- +/// index-fault convention `aura-campaign::exec::run_members` establishes for +/// the identical #272 seam), without needing a separate numeric-index +/// side-channel: the returned family already carries the points in order. +fn lowest_point_fault(family: &SweepFamily, faults: Mutex, String)>>) -> Option { + let captured = faults.into_inner().expect("fault capture lock"); + family + .points + .iter() + .find_map(|pt| captured.iter().find(|(p, _)| *p == pt.params).map(|(_, m)| m.clone())) +} + +/// Render + exit(3) on a contained member panic (#278: the deliberate +/// failed-cells exit, C14, never the raw 101 an uncontained panic would +/// yield) — reuses `aura_campaign::member_fault_prose`'s established +/// "a member panicked: …" wording rather than re-wording it, and the +/// `aura: warning: ` class marker (#278 decision) the CLI's own `diag` +/// macro would emit, hand-written here since that macro is presentation- +/// crate-private (#295 boundary) and this fires from `aura-runner`. +fn exit_on_member_panic(msg: &str) -> ! { + eprintln!("aura: warning: {}", member_fault_prose(&MemberFault::Panic(msg.to_string()))); + std::process::exit(3); +} + /// Validate the `--axis` grid against `doc`'s wrapped param space WITHOUT running any /// member (#253). Reuses the SAME strict, erroring axis-name check /// `blueprint_sweep_over` performs (`override_paths` — single-sourced, so the two @@ -426,15 +458,35 @@ pub fn blueprint_sweep_family( for (n, vals) in iter { binder = binder.axis(n, vals.clone()); } - binder + // #278: `run_blueprint_member` runs bare here (no #272 fault boundary of its + // own), so a member-compile panic (e.g. an `Sma::new` length assert) would + // otherwise unwind straight through this sweep to an uncaught exit 101 — + // contained the same way the real/campaign path contains it, via + // `catch_member_panic` + a per-point capture, resolved to the lowest + // enumeration index's message after the sweep joins (thread-order- + // independent, C1). + let faults: Mutex, String)>> = Mutex::new(Vec::new()); + let family = binder .sweep(|point| { // fresh per-member graph (Composite is !Clone, reload per member) run through // the shared reduce-mode member path — the same fn reproduction re-runs. - run_blueprint_member(reload(doc), point, &space, data.run_sources(env, &binding.columns()), window, 0, pip, &topo, env, DEFAULT_STOP, &binding, &[], NO_INSTRUMENT_CONTEXT) + match catch_member_panic(|| { + run_blueprint_member(reload(doc), point, &space, data.run_sources(env, &binding.columns()), window, 0, pip, &topo, env, DEFAULT_STOP, &binding, &[], NO_INSTRUMENT_CONTEXT) + }) { + Ok(report) => report, + Err(msg) => { + faults.lock().expect("fault capture lock").push((point.to_vec(), msg)); + axis_grid_probe_report() + } + } }) // render the sweep terminal's BindError to prose (#247), the fn's String error // contract — never the raw Debug struct. - .map_err(|e| render_bind_error(&e)) + .map_err(|e| render_bind_error(&e))?; + if let Some(msg) = lowest_point_fault(&family, faults) { + exit_on_member_panic(&msg); + } + Ok(family) } /// Sweep the LOADED blueprint over the user `--axis` grid on an in-sample window @@ -447,10 +499,19 @@ pub fn blueprint_sweep_family( /// its walk-forward in-sample twin); an axis matching neither space is refused /// (wrapped as `BindError::UnknownKnob`, the honest replacement for the retired /// "fully bound; nothing to sweep" refusal) before any member runs. +/// +/// The returned `Option` (#278) is this fn's OWN resolved member-panic +/// capture: this fn's nested sweep runs inside `blueprint_walkforward_family`'s +/// per-window parallel closure, so a contained member panic here must NOT +/// `eprintln!`+`exit` on the spot (that would race across windows, the exact +/// `#177` duplicated-rejection class the dispatch-boundary axis pre-flight +/// above already avoids) — captured internally, resolved to the lowest-index +/// message via [`lowest_point_fault`] before returning, for the caller to push +/// onto its own per-window fault list only after every window has joined. pub fn blueprint_sweep_over( doc: &str, axes: &[(String, Vec)], from: Timestamp, to: Timestamp, data: &DataSource, env: &Env, binding: &ResolvedBinding, -) -> Result<(SweepFamily, Vec), BindError> { +) -> Result<(SweepFamily, Vec, Option), BindError> { let reload = |d: &str| { blueprint_from_json(d, &|t| env.resolve(t)) .expect("doc parse-validated at the dispatch boundary; reload is infallible") @@ -476,11 +537,22 @@ pub fn blueprint_sweep_over( for (n, vals) in iter { binder = binder.axis(n, vals.clone()); } - binder.sweep_with_lattice(|point| { + let faults: Mutex, String)>> = Mutex::new(Vec::new()); + let (family, lattice) = binder.sweep_with_lattice(|point| { let sources = data.windowed_sources(from, to, env, &binding.columns()); let window = window_of(&sources).expect("non-empty in-sample window"); - run_blueprint_member(reopen_all(reload(doc), &overrides), point, &space, sources, window, 0, pip, &topo, env, DEFAULT_STOP, binding, &[], NO_INSTRUMENT_CONTEXT) - }) + match catch_member_panic(|| { + run_blueprint_member(reopen_all(reload(doc), &overrides), point, &space, sources, window, 0, pip, &topo, env, DEFAULT_STOP, binding, &[], NO_INSTRUMENT_CONTEXT) + }) { + Ok(report) => report, + Err(msg) => { + faults.lock().expect("fault capture lock").push((point.to_vec(), msg)); + axis_grid_probe_report() + } + } + })?; + let fault = lowest_point_fault(&family, faults); + Ok((family, lattice, fault)) } /// Run the winner params over an out-of-sample window `[from,to]` on the loaded @@ -509,6 +581,37 @@ pub fn run_oos_blueprint( (Vec::new(), report) } +/// A discarded placeholder [`WindowRun`] for a window whose IS refit or OOS +/// run was contained after a member panic (#278): `chosen_params` matches +/// `space`'s arity with inert zero cells (the `assemble_walk_forward` arity +/// invariant), `oos_equity` is empty (an empty segment leaves the stitched +/// curve unbroken, same convention as a genuinely equity-less window), and +/// the report reuses [`axis_grid_probe_report`]'s zero-cost placeholder. +/// Never rendered: the caller always resolves the captured window fault and +/// exits before touching a faulted window's run. +fn placeholder_window_run(space: &[ParamSpec]) -> WindowRun { + WindowRun { + chosen_params: space.iter().map(|_| Cell::from_i64(0)).collect(), + oos_equity: Vec::new(), + oos_report: axis_grid_probe_report(), + } +} + +/// The LOWEST roll-order window's captured member-panic message (#278), +/// found by walking `result.windows` — guaranteed by `assemble_walk_forward` +/// to be in roll order regardless of thread completion (C1) — for the first +/// window whose bounds match a captured fault. Thread-order-independent by +/// construction, mirroring [`lowest_point_fault`]'s identical convention one +/// level down (per-point within one window's IS sweep) and +/// `aura-campaign::exec`'s own lowest-index walk-forward fault attribution. +fn lowest_window_fault(result: &WalkForwardResult, faults: Mutex>) -> Option { + let captured = faults.into_inner().expect("fault capture lock"); + result + .windows + .iter() + .find_map(|w| captured.iter().find(|(b, _)| *b == w.bounds).map(|(_, m)| m.clone())) +} + /// The loaded-blueprint IS-refit walk-forward: per IS window, re-optimize the /// blueprint over the user `--axis` grid, select by `sqn_normalized`, run the /// winner OOS, reusing the generic `walk_forward` driver + `select_winner`; @@ -568,18 +671,45 @@ pub fn blueprint_walkforward_family( eprintln!("aura: {}", render_bind_error(&e)); std::process::exit(2); } - walk_forward(roller, space.clone(), |w: WindowBounds| { - let (is_family, lattice) = blueprint_sweep_over(doc, axes, w.is.0, w.is.1, data, env, &binding) + // #278: `blueprint_sweep_over`'s inner sweep and `run_oos_blueprint`'s single + // member run both drive `run_blueprint_member` bare, so either could + // otherwise unwind an uncaught member-compile panic straight through this + // window closure. Both are contained (`blueprint_sweep_over` resolves its + // own capture and returns it; `run_oos_blueprint` via `catch_member_panic` + // directly) and captured PER WINDOW here rather than printed on the spot — + // this closure runs across windows in parallel (`walk_forward`), so an + // inline `eprintln!`+`exit` would race the same way the axis pre-flight + // above was written to avoid (#177). The lowest roll-order window's fault + // is resolved once, after every window has joined. + let window_faults: Mutex> = Mutex::new(Vec::new()); + let result = walk_forward(roller, space.clone(), |w: WindowBounds| { + let (is_family, lattice, is_fault) = blueprint_sweep_over(doc, axes, w.is.0, w.is.1, data, env, &binding) .expect("axes validated in the dispatch-boundary pre-flight"); + if let Some(msg) = is_fault { + window_faults.lock().expect("fault capture lock").push((w, msg)); + return placeholder_window_run(&space); + } let (best, selection) = match select_winner(&is_family, WINNER_SELECTION_METRIC, select, Some(&lattice)) { Ok(v) => v, Err(msg) => { eprintln!("aura: {msg}"); std::process::exit(2); } }; - let (oos_equity, mut oos_report) = - run_oos_blueprint(doc, &best.params, &space, w.oos.0, w.oos.1, &topo, data, env, &binding, &overrides); - oos_report.manifest.selection = Some(selection); - WindowRun { chosen_params: best.params, oos_equity, oos_report } - }) + match catch_member_panic(|| { + run_oos_blueprint(doc, &best.params, &space, w.oos.0, w.oos.1, &topo, data, env, &binding, &overrides) + }) { + Ok((oos_equity, mut oos_report)) => { + oos_report.manifest.selection = Some(selection); + WindowRun { chosen_params: best.params, oos_equity, oos_report } + } + Err(msg) => { + window_faults.lock().expect("fault capture lock").push((w, msg)); + placeholder_window_run(&space) + } + } + }); + if let Some(msg) = lowest_window_fault(&result, window_faults) { + exit_on_member_panic(&msg); + } + result } /// A fresh seeded synthetic price walk for one Monte-Carlo draw — `blueprint_mc_family`'s