diff --git a/crates/aura-campaign/src/exec.rs b/crates/aura-campaign/src/exec.rs index 14bdf9a..0466ae9 100644 --- a/crates/aura-campaign/src/exec.rs +++ b/crates/aura-campaign/src/exec.rs @@ -18,9 +18,8 @@ use aura_engine::{ }; use aura_registry::{ derive_trace_name, generalization, optimize, optimize_deflated, optimize_plateau, - sweep_member_reports, walkforward_member_reports, CampaignGeneralization, CampaignRunRecord, - CellRealization, FamilyKind, PlateauMode, Registry, StageBootstrap, StageRealization, - StageSelection, + sweep_member_reports, CampaignGeneralization, CampaignRunRecord, CellRealization, FamilyKind, + PlateauMode, Registry, StageBootstrap, StageRealization, StageSelection, }; use aura_research::{Axis, CampaignDoc, DocRef, ProcessDoc, SelectRule, StageBlock, WfMode}; @@ -290,6 +289,7 @@ fn run_cell( survivor_ordinals: None, selection: None, bootstrap: None, + window_faults: vec![], }); } StageBlock::Sweep { selection } => { @@ -299,7 +299,14 @@ fn run_cell( "{campaign_prefix}-{}-{}-w{window_ordinal}-r{}-s{stage_ordinal}", cell.strategy_ordinal, cell.instrument, cell.regime_ordinal, ); - let family = run_members(cell, &grid, runner)?; + let family = match run_members(cell, &grid, runner) { + Ok(family) => family, + Err(fault) => { + // Registry etc. still aborts (contain's Err arm). + let cf = contain(fault, stage_ordinal)?; + return Ok(failed_cell(cell, stages, families, selections, cf, runner)); + } + }; let family_id = registry .append_family(&family_name, FamilyKind::Sweep, &sweep_member_reports(&family)) .map_err(ExecFault::Registry)?; @@ -336,6 +343,7 @@ fn run_cell( survivor_ordinals: None, selection: Some(StageSelection { winner_ordinal, params, selection }), bootstrap: None, + window_faults: vec![], }); } None => { @@ -349,6 +357,7 @@ fn run_cell( survivor_ordinals: None, selection: None, bootstrap: None, + window_faults: vec![], }); } } @@ -398,6 +407,7 @@ fn run_cell( survivor_ordinals: Some(ordinals), selection: None, bootstrap: None, + window_faults: vec![], }); if empty { nominee = None; // a truncated cell nominates no candidate @@ -415,7 +425,7 @@ fn run_cell( // re-sweeps the surviving param points and never needs the // gate bookkeeping's ordinals or reports. let survivor_points: Vec> = flow.points(); - let fam = run_walk_forward_stage( + match run_walk_forward_stage( seed, cell, stage_ordinal, @@ -426,22 +436,31 @@ fn run_cell( &family_name, runner, registry, - )?; - stages.push(StageRealization { - block: "std::walk_forward".to_string(), - family_id: Some(fam.family_id.clone()), - survivor_ordinals: None, - selection: None, - bootstrap: None, - }); - // Nominate the LAST window's OOS report (roll order per - // walkforward_member_reports) with its chosen params — the wf - // stage stamps them on the fresh OOS report's manifest. - nominee = fam - .reports - .last() - .map(|last| (last.manifest.params.clone(), last.clone())); - families.push(fam); + ) { + Ok(WfStageOutcome::Ran { family: fam, window_faults }) => { + stages.push(StageRealization { + block: "std::walk_forward".to_string(), + family_id: Some(fam.family_id.clone()), + survivor_ordinals: None, + selection: None, + bootstrap: None, + window_faults, + }); + // Nominate the LAST window's OOS report (roll order, + // over the surviving folds) with its chosen params — + // the wf stage stamps them on the fresh OOS report's + // manifest. + nominee = fam + .reports + .last() + .map(|last| (last.manifest.params.clone(), last.clone())); + families.push(fam); + } + Ok(WfStageOutcome::CellFailed(cf)) => { + return Ok(failed_cell(cell, stages, families, selections, cf, runner)); + } + Err(e) => return Err(e), // Registry / global refusal + } } StageBlock::MonteCarlo { resamples, block_len } => { // Terminal annotator (#200 d1/d3): nothing flows out. Dual @@ -500,6 +519,7 @@ fn run_cell( survivor_ordinals: None, selection: None, bootstrap: Some(bootstrap), + window_faults: vec![], }); } StageBlock::Generalize { .. } => { @@ -518,6 +538,8 @@ fn run_cell( window_ms: cell.window_ms, regime: cell.regime, regime_ordinal: cell.regime_ordinal, + fault: None, + coverage: runner.window_coverage(cell), stages, }, )) @@ -556,6 +578,54 @@ fn enumerate_grid(axes: &BTreeMap) -> SweepGrid { SweepGrid { specs, axis_lens, points } } +/// Ref-counted panic-hook silencer: while at least one `SilencedPanic` guard +/// is alive, the process's panic hook is a no-op, so a member panic CONTAINED +/// by one of this module's three `catch_unwind` sites (recorded as a +/// `MemberFault`/`WindowFault`, the run continuing past it) does not also +/// paint a crash-looking default backtrace to stderr — "recorded, campaign +/// continues" stays observably true, not just true in the record. Ref-counted +/// rather than a bare set/restore because `sweep`/`walk_forward` run these +/// closures on multiple OS threads concurrently (C1 disjoint-parallel +/// members): a naive per-closure set_hook/take_hook pair would race two +/// threads' install/restore against each other. The previous hook (whatever +/// it was — default or a caller's own) is saved on the 0→1 transition and +/// restored on the 1→0 transition, so this never permanently clobbers a +/// hook installed above this module. +struct SilencedPanic; + +#[allow(clippy::type_complexity)] +static PANIC_HOOK_STATE: Mutex<(usize, Option) + Sync + Send>>)> = + Mutex::new((0, None)); + +impl SilencedPanic { + /// Enter the silenced section; drop the returned guard to leave it. Hold + /// this only around the `catch_unwind` call itself, never the whole + /// member-run closure, so a genuinely uncontained escape (this module's + /// own bug, not the member's) still surfaces normally once the guard has + /// dropped and unwinding continues past `catch_unwind`. + fn enter() -> Self { + let mut state = PANIC_HOOK_STATE.lock().expect("panic hook state lock"); + if state.0 == 0 { + state.1 = Some(std::panic::take_hook()); + std::panic::set_hook(Box::new(|_| {})); + } + state.0 += 1; + SilencedPanic + } +} + +impl Drop for SilencedPanic { + fn drop(&mut self) { + let mut state = PANIC_HOOK_STATE.lock().expect("panic hook state lock"); + state.0 -= 1; + if state.0 == 0 + && let Some(prev) = state.1.take() + { + std::panic::set_hook(prev); + } + } +} + /// Run every grid point through the consumer's `MemberRunner` via the engine /// `sweep` (disjoint parallel members, C1 enumeration order). Faults are /// captured per slot (the closure must return a report), and the LOWEST @@ -582,9 +652,15 @@ fn run_members( let faults: Mutex> = Mutex::new(Vec::new()); let family = sweep(&space, |point: &[Cell]| { let params = zip_params(&grid.specs, point); - match runner.run_member(cell, ¶ms, cell.window_ms) { - Ok(report) => report, - Err(fault) => { + let called = { + let _silence = SilencedPanic::enter(); + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + runner.run_member(cell, ¶ms, cell.window_ms) + })) + }; + match called { + Ok(Ok(report)) => report, + Ok(Err(fault)) => { let index = cells .iter() .position(|c| c.as_slice() == point) @@ -592,6 +668,17 @@ fn run_members( faults.lock().expect("fault capture lock").push((index, fault)); placeholder_report(¶ms, cell.window_ms) } + Err(payload) => { + let index = cells + .iter() + .position(|c| c.as_slice() == point) + .expect("the sweep only enumerates the grid's own points"); + faults + .lock() + .expect("fault capture lock") + .push((index, MemberFault::Panic(panic_message(payload)))); + placeholder_report(¶ms, cell.window_ms) + } } }); let mut captured = faults.into_inner().expect("fault capture lock"); @@ -602,6 +689,97 @@ fn run_members( Ok(family) } +/// #272: which executor faults are per-cell outcomes vs campaign-wide +/// refusals. Member and window faults are data-and-cell-local; everything else +/// (doc-shape, registry) stays a global refusal returned to the caller. +fn contain(fault: ExecFault, stage: usize) -> Result { + match fault { + ExecFault::Member(m) => Ok(aura_registry::CellFault { + stage, + kind: member_kind(&m), + detail: member_fault_prose(&m), + }), + ExecFault::Window { stage: s, detail } => Ok(aura_registry::CellFault { + stage: s, + kind: aura_registry::CellFaultKind::Window, + detail, + }), + other => Err(other), + } +} + +fn member_kind(m: &MemberFault) -> aura_registry::CellFaultKind { + use aura_registry::CellFaultKind as K; + match m { + MemberFault::NoData { .. } => K::NoData, + MemberFault::Bind(_) => K::Bind, + MemberFault::Run(_) => K::Run, + MemberFault::Panic(_) => K::Panic, + } +} + +/// Prose for a member fault. `pub` (re-exported at the crate root) so the +/// CLI's `exec_fault_prose` phrases the `ExecFault::Member` arm by calling +/// this directly instead of carrying its own duplicate of the wording. +/// Debug-leak-free, self-describing. +pub fn member_fault_prose(m: &MemberFault) -> String { + match m { + MemberFault::NoData { instrument, window_ms } => format!( + "no data for instrument {instrument} in window [{}, {}] (epoch-ms)", + window_ms.0, window_ms.1 + ), + MemberFault::Bind(detail) => format!("a member failed to bind: {detail}"), + MemberFault::Run(detail) => format!("a member failed to run: {detail}"), + MemberFault::Panic(detail) => format!("a member panicked: {detail}"), + } +} + +/// Best-effort panic-payload message (String / &str payloads; else a fixed +/// marker). `AssertUnwindSafe` at the call sites is a documented judgement: the +/// runner is reused for later cells; the CLI runner holds read-only state. +fn panic_message(payload: Box) -> String { + payload + .downcast_ref::() + .cloned() + .or_else(|| payload.downcast_ref::<&str>().map(|s| s.to_string())) + .unwrap_or_else(|| "member panicked (non-string payload)".to_string()) +} + +/// Assemble the (CellOutcome, CellRealization) pair for a cell that failed at a +/// stage (#272): the realized stage prefix survives in `stages`, `families`/ +/// `selections` carry whatever already-persisted stages produced (empty for a +/// sweep-first fault; the sweep family/selection for a later wf-stage fault), +/// no nominee, the fault stamped on both, coverage from the runner. +fn failed_cell( + cell: &CellSpec, + stages: Vec, + families: Vec, + selections: Vec, + fault: aura_registry::CellFault, + runner: &dyn MemberRunner, +) -> (CellOutcome, CellRealization) { + ( + CellOutcome { families, selections, nominee: None }, + CellRealization { + strategy: cell.strategy_id.clone(), + instrument: cell.instrument.clone(), + window_ms: cell.window_ms, + stages, + regime: cell.regime, + regime_ordinal: cell.regime_ordinal, + fault: Some(fault), + coverage: runner.window_coverage(cell), + }, + ) +} + +/// Sentinel `RunManifest::broker` value stamped onto [`placeholder_report`]'s +/// output — the single source both the setter and the survivor filter +/// (`run_members`'s `... != FAULTED_MEMBER_PLACEHOLDER_BROKER` sibling) read, +/// so the two can never drift apart into silently pooling a faulted +/// placeholder into a real family/OOS selection. +const FAULTED_MEMBER_PLACEHOLDER_BROKER: &str = "faulted-member-placeholder"; + /// Slot filler for a faulted member: the engine sweep contract needs one /// report per slot, but a captured fault aborts `execute` before any family /// write, so this report is never persisted and never ranked. @@ -613,7 +791,7 @@ fn placeholder_report(params: &[(String, Scalar)], window_ms: (i64, i64)) -> Run defaults: Vec::new(), window: (Timestamp(window_ms.0), Timestamp(window_ms.1)), seed: 0, - broker: "faulted-member-placeholder".to_string(), + broker: FAULTED_MEMBER_PLACEHOLDER_BROKER.to_string(), selection: None, instrument: None, topology_hash: None, @@ -685,8 +863,9 @@ fn scalar_point(space: &[ParamSpec], point: &[Cell]) -> Vec { /// The pooled walk-forward OOS trade-R series: the family reports' `net_trade_rs` /// concatenated in report order — which IS roll order (the wf stage builds its -/// family via `walkforward_member_reports`, per-window OOS reports in roll -/// order). Reports without an R block contribute nothing. +/// family via `walkforward_member_reports_of`, per-window OOS reports in roll +/// order, over the surviving folds). Reports without an R block contribute +/// nothing. fn pooled_net_trade_rs(reports: &[RunReport]) -> Vec { let mut pooled = Vec::new(); for report in reports { @@ -734,7 +913,7 @@ fn run_walk_forward_stage( family_name: &str, runner: &dyn MemberRunner, registry: &Registry, -) -> Result { +) -> Result { let StageBlock::WalkForward { in_sample_ms, out_of_sample_ms, step_ms, mode, metric, select } = block else { @@ -813,9 +992,16 @@ fn run_walk_forward_stage( // with the same member fault capture the sweep stage uses. let is_faults: Mutex> = Mutex::new(Vec::new()); let is_family = sweep(&is_space, |cells| { - match runner.run_member(cell, &zip_params(specs, cells), (w.is.0 .0, w.is.1 .0)) { - Ok(report) => report, - Err(fault) => { + let params = zip_params(specs, cells); + let called = { + let _silence = SilencedPanic::enter(); + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + runner.run_member(cell, ¶ms, (w.is.0 .0, w.is.1 .0)) + })) + }; + match called { + Ok(Ok(report)) => report, + Ok(Err(fault)) => { let midx = is_points .iter() .position(|p| p.as_slice() == cells) @@ -823,6 +1009,14 @@ fn run_walk_forward_stage( is_faults.lock().unwrap().push((midx, fault)); placeholder_report(&[], (0, 0)) } + Err(payload) => { + let midx = is_points + .iter() + .position(|p| p.as_slice() == cells) + .expect("sweep enumerates exactly is_points"); + is_faults.lock().unwrap().push((midx, MemberFault::Panic(panic_message(payload)))); + placeholder_report(&[], (0, 0)) + } } }); let member_faults = is_faults.into_inner().expect("no thread panicked holding the lock"); @@ -845,29 +1039,93 @@ fn run_walk_forward_stage( // OOS: run the winner over the out-of-sample bounds; the selection is // stamped on the fresh OOS report (a new record, never a mutation of // a stored one). - let mut oos_report = match runner.run_member( - cell, - &zip_params(specs, &winner.params), - (w.oos.0 .0, w.oos.1 .0), - ) { - Ok(report) => report, - Err(fault) => { + let oos_params = zip_params(specs, &winner.params); + let oos_called = { + let _silence = SilencedPanic::enter(); + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + runner.run_member(cell, &oos_params, (w.oos.0 .0, w.oos.1 .0)) + })) + }; + let mut oos_report = match oos_called { + Ok(Ok(report)) => report, + Ok(Err(fault)) => { faults.lock().unwrap().push((widx, ExecFault::Member(fault))); return placeholder_window_run(specs); } + Err(payload) => { + faults + .lock() + .unwrap() + .push((widx, ExecFault::Member(MemberFault::Panic(panic_message(payload))))); + return placeholder_window_run(specs); + } }; oos_report.manifest.selection = Some(selection); WindowRun { chosen_params: winner.params, oos_equity: vec![], oos_report } }); - let window_faults = faults.into_inner().expect("no thread panicked holding the lock"); - if let Some((_, fault)) = window_faults.into_iter().min_by_key(|(i, _)| *i) { - return Err(fault); + // Registry faults (optimizer/persistence broken) stay a global abort; + // Member/Window faults become per-fold WindowFaults via the same + // `contain` mapping the cell-level containment uses. One drain, in + // ascending window-index order — ExecFault/RegistryError are not Clone, + // so this is a single ownership-respecting pass rather than iter()-then- + // into_iter(); the first Registry fault encountered (ascending order = + // lowest index) wins, matching the existing lowest-index-fault convention. + let mut captured = faults.into_inner().expect("no thread panicked holding the lock"); + captured.sort_by_key(|(idx, _)| *idx); + let mut window_faults: Vec = Vec::with_capacity(captured.len()); + for (widx, fault) in captured { + match contain(fault, stage) { + Ok(cf) => window_faults.push(aura_registry::WindowFault { + window_ordinal: widx, + kind: cf.kind, + detail: cf.detail, + }), + Err(global) => return Err(global), + } } - let reports = walkforward_member_reports(&result); + // Keep only real OOS reports (drop faulted-fold placeholders by the broker + // sentinel) before building the family. + let surviving: Vec<&aura_engine::WindowOutcome> = result + .windows + .iter() + .filter(|w| w.run.oos_report.manifest.broker != FAULTED_MEMBER_PLACEHOLDER_BROKER) + .collect(); + if surviving.is_empty() { + // Every fold failed — the cell fails at this stage. + let first = window_faults.first().cloned(); + return Ok(WfStageOutcome::CellFailed(aura_registry::CellFault { + stage, + kind: first.as_ref().map(|w| w.kind).unwrap_or(aura_registry::CellFaultKind::Run), + detail: format!( + "all {} walk_forward folds failed{}", + result.windows.len(), + first.map(|w| format!("; first: {}", w.detail)).unwrap_or_default() + ), + })); + } + let reports = walkforward_member_reports_of(&surviving); let family_id = registry .append_family(family_name, FamilyKind::WalkForward, &reports) .map_err(ExecFault::Registry)?; - Ok(StageFamily { stage, block: "std::walk_forward", family_id, reports }) + Ok(WfStageOutcome::Ran { + family: StageFamily { stage, block: "std::walk_forward", family_id, reports }, + window_faults, + }) +} + +/// #272: a wf stage either ran (its family + any per-fold holes) or the whole +/// cell failed (every fold faulted). A Registry fault is a separate global Err +/// (returned directly from `run_walk_forward_stage`, never wrapped here). +enum WfStageOutcome { + Ran { family: StageFamily, window_faults: Vec }, + CellFailed(aura_registry::CellFault), +} + +/// The surviving (non-faulted) walk-forward windows' OOS reports, in roll +/// order — the same mapping `walkforward_member_reports` does over a full +/// result, applied to the filtered survivor slice. +fn walkforward_member_reports_of(windows: &[&aura_engine::WindowOutcome]) -> Vec { + windows.iter().map(|w| w.run.oos_report.clone()).collect() } diff --git a/crates/aura-campaign/src/lib.rs b/crates/aura-campaign/src/lib.rs index 876ee8d..b430738 100644 --- a/crates/aura-campaign/src/lib.rs +++ b/crates/aura-campaign/src/lib.rs @@ -12,7 +12,9 @@ //! and no UI. mod exec; -pub use exec::{execute, CampaignOutcome, CellOutcome, StageFamily, StageSelectionOut}; +pub use exec::{ + execute, member_fault_prose, CampaignOutcome, CellOutcome, StageFamily, StageSelectionOut, +}; use std::collections::BTreeMap; @@ -54,6 +56,13 @@ pub trait MemberRunner: Sync { params: &[(String, Scalar)], window_ms: (i64, i64), ) -> Result; + + /// The archive coverage of this cell's (instrument, window), when the + /// runner can know it and it deviates from the requested window (#272). + /// One call per cell — coverage is a property of the cell, not a member. + fn window_coverage(&self, _cell: &CellSpec) -> Option { + None + } } /// Display-free member faults (the consumer phrases them — the RefFault @@ -63,6 +72,9 @@ pub enum MemberFault { NoData { instrument: String, window_ms: (i64, i64) }, Bind(String), Run(String), + /// A panic that unwound out of `MemberRunner::run_member`, caught at the + /// member boundary (#272) — the payload's best-effort message. + Panic(String), } /// Preflight + runtime refusals. Display-free and by-identifier: the @@ -1246,75 +1258,161 @@ mod wf_tests { } } - /// A member fault raised INSIDE a window's in-sample sweep aborts the - /// walk_forward stage with that fault, and among several faulted windows - /// the LOWEST roll index wins (the per-window `min_by_key` attribution - /// mirrors the sweep stage's own lowest-enumeration-index convention) — - /// never the faulted stage's family, since a captured fault returns - /// before any registry write. + /// #272: a member fault raised INSIDE one window's in-sample sweep is + /// contained as that fold's `WindowFault` — the surviving windows still + /// run, pool into the walk_forward family, and the campaign run record + /// persists. Faulting w1's IS run out of 3 windows (w0, w1, w2) leaves + /// exactly 2 surviving OOS reports and one recorded window fault at + /// ordinal 1. #[test] - fn wf_is_member_fault_lowest_window_wins() { + fn wf_one_faulted_fold_is_contained_survivors_pool() { let registry = wf_registry("is-fault"); let campaign = wf_campaign((0, 99)); let process = wf_process(None, 40, 20, 20); - // IS bounds: w0 (0,39), w1 (20,59), w2 (40,79) — fault both w1's and - // w2's IS runs; the widx-1 fault must win over widx-2's. + // IS bounds: w0 (0,39), w1 (20,59), w2 (40,79) — fault only w1's IS + // run; w0 and w2 survive and pool. + let runner = WfFakeRunner::faulty(vec![( + (20, 59), + vec![("len".to_string(), Scalar::i64(4))], + MemberFault::Run("is-w1".to_string()), + )]); + let outcome = run_wf(&campaign, &process, &runner, ®istry) + .expect("a faulted fold is contained, not a stage abort"); + + let fam = outcome.cells[0] + .families + .iter() + .find(|f| f.block == "std::walk_forward") + .expect("wf family present despite the faulted fold"); + assert_eq!(fam.reports.len(), 2, "only the 2 surviving windows pool"); + + let cell = &outcome.record.cells[0]; + let wf_stage = cell + .stages + .iter() + .find(|s| s.block == "std::walk_forward") + .expect("wf stage realized"); + assert_eq!(wf_stage.window_faults.len(), 1); + assert_eq!(wf_stage.window_faults[0].window_ordinal, 1); + assert_eq!(wf_stage.window_faults[0].kind, aura_registry::CellFaultKind::Run); + assert!(cell.fault.is_none(), "a partial-fold stage does not fail the cell"); + + assert_eq!(registry.load_campaign_runs().expect("load campaign runs").len(), 1); + let members = registry.load_family_members().expect("load members"); + assert!(members.iter().any(|m| m.kind == FamilyKind::WalkForward)); + } + + /// #272: every fold faulting (both the IS sweep AND the OOS run, across + /// all 3 windows) leaves zero surviving OOS reports — the whole cell + /// fails at the wf stage, with no WalkForward family persisted, while the + /// campaign run record still records the failed cell. + #[test] + fn wf_all_folds_faulted_fails_the_cell() { + let registry = wf_registry("all-folds-fault"); + let campaign = wf_campaign((0, 99)); + let process = wf_process(None, 40, 20, 20); + // IS bounds: w0 (0,39), w1 (20,59), w2 (40,79) — fault every window's + // IS run so no window reaches its OOS call. let runner = WfFakeRunner::faulty(vec![ + ( + (0, 39), + vec![("len".to_string(), Scalar::i64(4))], + MemberFault::Run("is-w0".to_string()), + ), ( (20, 59), - vec![("len".to_string(), Scalar::i64(3))], + vec![("len".to_string(), Scalar::i64(4))], MemberFault::Run("is-w1".to_string()), ), ( (40, 79), - vec![("len".to_string(), Scalar::i64(2))], + vec![("len".to_string(), Scalar::i64(4))], MemberFault::Run("is-w2".to_string()), ), ]); - let err = run_wf(&campaign, &process, &runner, ®istry) - .expect_err("a faulted IS member aborts the walk_forward stage"); - match err { - ExecFault::Member(fault) => assert_eq!( - fault, - MemberFault::Run("is-w1".to_string()), - "the LOWEST window index's fault wins, not w2's later one", - ), - other => panic!("expected ExecFault::Member, got {other:?}"), - } + let outcome = run_wf(&campaign, &process, &runner, ®istry) + .expect("an all-folds-faulted stage fails the cell, not the process"); + + assert!( + outcome.cells[0].families.iter().all(|f| f.block != "std::walk_forward"), + "no WalkForward family is produced when every fold fails" + ); + let cell = &outcome.record.cells[0]; + let fault = cell.fault.as_ref().expect("the cell fails at the wf stage"); + assert_eq!(fault.stage, 1, "walk_forward is pipeline stage 1 here"); + assert!(fault.detail.contains("all 3 walk_forward folds failed")); + + assert_eq!(registry.load_campaign_runs().expect("load campaign runs").len(), 1); let members = registry.load_family_members().expect("load members"); assert!(members.iter().all(|m| m.kind != FamilyKind::WalkForward)); - assert!(registry.load_campaign_runs().expect("load campaign runs").is_empty()); } - /// A member fault raised running the IS winner over its OUT-of-sample - /// bounds (a call distinct from every IS-sweep call, since the OOS - /// window's bounds differ from every window's IS bounds) also aborts the - /// walk_forward stage with that fault, and — like the IS-sweep path — - /// never persists the faulted stage's family. + /// #272: a member PANIC inside a window's in-sample sweep is caught at the + /// wf member boundary (`catch_unwind`) and contained as that fold's + /// `WindowFault` with kind `panic` — the process does not abort, the + /// surviving folds pool, exactly as an `Err` fault is contained. Guards the + /// wf-path panic arm (the sweep-path arm is covered by `PanicRunner` in + /// tests/execute.rs; this is its walk_forward mirror). #[test] - fn wf_oos_member_fault_aborts_walk_forward() { - let registry = wf_registry("oos-fault"); + fn wf_member_panic_in_a_fold_is_contained() { + // Panics on w1's IS bounds (20,59); every other call returns a report. + struct WfPanicRunner; + impl MemberRunner for WfPanicRunner { + fn run_member( + &self, + _cell: &CellSpec, + params: &[(String, Scalar)], + window_ms: (i64, i64), + ) -> Result { + assert_ne!(window_ms, (20, 59), "planted panic on w1's in-sample run"); + let total: f64 = params + .iter() + .map(|(_, s)| match s { + Scalar::I64(v) => *v as f64, + Scalar::F64(v) => *v, + _ => 0.0, + }) + .sum(); + Ok(RunReport { + manifest: RunManifest { + commit: "wf-panic-fake".to_string(), + params: params.to_vec(), + defaults: Vec::new(), + window: (Timestamp(window_ms.0), Timestamp(window_ms.1)), + seed: 0, + broker: "fake".to_string(), + selection: None, + instrument: None, + topology_hash: None, + project: None, + }, + metrics: RunMetrics { + total_pips: total, + max_drawdown: 0.0, + bias_sign_flips: 0, + r: None, + }, + }) + } + } + let registry = wf_registry("is-panic"); let campaign = wf_campaign((0, 99)); let process = wf_process(None, 40, 20, 20); - // total_pips == len, so the IS winner is always len=4 (argmax over - // [1,2,3,4]); fault only window 0's OOS run of that winner (OOS - // bounds (40,59) match no window's IS bounds and no other window's - // OOS bounds). - let runner = WfFakeRunner::faulty(vec![( - (40, 59), - vec![("len".to_string(), Scalar::i64(4))], - MemberFault::Run("oos-w0".to_string()), - )]); - let err = run_wf(&campaign, &process, &runner, ®istry) - .expect_err("a faulted OOS member aborts the walk_forward stage"); - match err { - ExecFault::Member(fault) => { - assert_eq!(fault, MemberFault::Run("oos-w0".to_string())) - } - other => panic!("expected ExecFault::Member, got {other:?}"), - } - let members = registry.load_family_members().expect("load members"); - assert!(members.iter().all(|m| m.kind != FamilyKind::WalkForward)); + let strategies = vec![("s".repeat(64), "{}".to_string())]; + let outcome = execute(&"e".repeat(64), &campaign, &process, &strategies, &WfPanicRunner, ®istry) + .expect("a member panic in a fold is contained, not a process abort"); + + let cell = &outcome.record.cells[0]; + let wf_stage = cell + .stages + .iter() + .find(|s| s.block == "std::walk_forward") + .expect("wf stage realized despite the panicked fold"); + assert_eq!(wf_stage.window_faults.len(), 1); + assert_eq!(wf_stage.window_faults[0].window_ordinal, 1); + assert_eq!(wf_stage.window_faults[0].kind, aura_registry::CellFaultKind::Panic); + assert!(cell.fault.is_none(), "a partial-fold stage does not fail the cell"); + assert_eq!(registry.load_campaign_runs().expect("load campaign runs").len(), 1); } /// select_sweep_winner's Plateau arms dispatch through the public diff --git a/crates/aura-campaign/tests/execute.rs b/crates/aura-campaign/tests/execute.rs index aa42501..68d6b40 100644 --- a/crates/aura-campaign/tests/execute.rs +++ b/crates/aura-campaign/tests/execute.rs @@ -126,6 +126,22 @@ impl MemberRunner for ZeroTradeRunner { } } +/// A runner whose `run_member` always panics — the #272 panic-containment +/// fixture (`catch_unwind` at the member boundary must turn this into a +/// `MemberFault::Panic`, never an unwind out of `execute`). +struct PanicRunner; + +impl MemberRunner for PanicRunner { + fn run_member( + &self, + _cell: &CellSpec, + _params: &[(String, Scalar)], + _window_ms: (i64, i64), + ) -> Result { + panic!("planted member panic") + } +} + /// fast in {2,3}, slow in {6,9} -> 4 odometer points (last axis fastest): /// 0:(2,6) net .26 | 1:(2,9) net .29 | 2:(3,6) net .36 | 3:(3,9) net .39. fn axes_2x2() -> BTreeMap { @@ -400,34 +416,57 @@ fn execute_is_deterministic_twice() { assert_eq!(runs[1], second.record); } +/// #272: a member fault is contained as a failed cell, not a global abort — +/// the campaign run record still persists (the whole point of containment), +/// while a sweep-faulted cell writes no family (`run_members` returns before +/// `append_family`). #[test] -fn execute_member_fault_aborts_with_lowest_index() { +fn execute_member_fault_records_a_failed_cell_and_continues() { let reg = temp_registry("member_fault"); - let doc = campaign(&["EURUSD"]); + // two instruments; the planted fault is keyed by param point, so both + // cells' sweeps hit it and both fail — the containment property under test + // is that execute returns Ok and records every cell, not a global abort. + let doc = campaign(&["AAA", "BBB"]); let proc_doc = process(vec![sweep_stage(false)]); - // faults planted at enumeration indices 1 (2,9) and 2 (3,6) let point = |fast: i64, slow: i64| { vec![("fast".to_string(), Scalar::i64(fast)), ("slow".to_string(), Scalar::i64(slow))] }; + // FakeRunner faults are matched by param point regardless of instrument; + // plant one on the grid so AAA's (and BBB's) sweep hit it — the whole + // sweep-faulted cell fails (grid hole compromises selection). let runner = FakeRunner { - faults: vec![ - (point(2, 9), MemberFault::Run("boom at index 1".to_string())), - (point(3, 6), MemberFault::Run("boom at index 2".to_string())), - ], + faults: vec![(point(2, 9), MemberFault::Run("boom".to_string()))], }; - let err = execute(CAMPAIGN_ID, &doc, &proc_doc, &strategies(), &runner, ®) - .expect_err("a faulted member aborts the run"); - match err { - ExecFault::Member(fault) => assert_eq!( - fault, - MemberFault::Run("boom at index 1".to_string()), - "the LOWEST enumeration index's fault is reported", - ), - other => panic!("expected ExecFault::Member, got {other:?}"), + let out = execute(CAMPAIGN_ID, &doc, &proc_doc, &strategies(), &runner, ®) + .expect("a faulted member is recorded, not a global abort"); + assert_eq!(out.record.cells.len(), 2, "both cells are recorded"); + for cell in &out.record.cells { + let fault = cell.fault.as_ref().expect("sweep-fault fails the cell"); + assert_eq!(fault.stage, 0); + assert_eq!(fault.kind, aura_registry::CellFaultKind::Run); } - // the fault aborted before any family or realization write + // The run record persisted (the whole point of containment); a sweep-fault + // cell writes no family (run_members returns before append_family). + assert_eq!(reg.load_campaign_runs().expect("load runs").len(), 1); assert!(reg.load_family_members().expect("load members").is_empty()); - assert!(reg.load_campaign_runs().expect("load campaign runs").is_empty()); +} + +/// #272: a panic that unwinds out of `MemberRunner::run_member` is caught at +/// the member boundary, not propagated — the cell is recorded failed with a +/// `Panic` kind, and the run record persists. +#[test] +fn execute_member_panic_is_contained_as_a_failed_cell() { + let reg = temp_registry("member_panic"); + let doc = campaign(&["AAA", "BBB"]); + let proc_doc = process(vec![sweep_stage(false)]); + let runner = PanicRunner; + let out = execute(CAMPAIGN_ID, &doc, &proc_doc, &strategies(), &runner, ®) + .expect("a member panic is contained, not a process abort"); + assert!(out.record.cells.iter().all(|c| c + .fault + .as_ref() + .is_some_and(|f| f.kind == aura_registry::CellFaultKind::Panic))); + assert_eq!(reg.load_campaign_runs().expect("load runs").len(), 1); } #[test] diff --git a/crates/aura-cli/src/campaign_run.rs b/crates/aura-cli/src/campaign_run.rs index 7fbd2dd..fcc4332 100644 --- a/crates/aura-cli/src/campaign_run.rs +++ b/crates/aura-cli/src/campaign_run.rs @@ -149,16 +149,7 @@ pub(crate) fn exec_fault_prose(f: &ExecFault) -> String { "process stage {stage}: walk_forward windows do not fit the campaign \ window: {detail}" ), - ExecFault::Member(MemberFault::NoData { instrument, window_ms }) => format!( - "no data for instrument {instrument} in window [{}, {}] (epoch-ms)", - window_ms.0, window_ms.1 - ), - ExecFault::Member(MemberFault::Bind(detail)) => { - format!("a member failed to bind: {detail}") - } - ExecFault::Member(MemberFault::Run(detail)) => { - format!("a member failed to run: {detail}") - } + ExecFault::Member(m) => aura_campaign::member_fault_prose(m), ExecFault::Registry(e) => e.to_string(), } } @@ -412,6 +403,62 @@ impl MemberRunner for CliMemberRunner<'_> { report.manifest.instrument = Some(cell.instrument.clone()); Ok(report) } + + /// #272: derive the cell's archive coverage from the #264 primitives — + /// `None` when the archive has no file for the instrument at all (the + /// `NoData` member fault's job, not coverage) or when the effective + /// evaluated span covers the requested window exactly with no interior + /// gap month. One call per cell (not per member): coverage is a property + /// of the cell's (instrument, window), never a swept param point. + fn window_coverage(&self, cell: &CellSpec) -> Option { + let data_path = PathBuf::from(self.env.data_path()); + let months = aura_ingest::list_m1_months(&data_path, &cell.instrument); + if months.is_empty() { + return None; // no archive at all is the NoData fault's job, not coverage + } + let (eff_from_ts, eff_to_ts) = aura_ingest::archive_extent( + &self.server, + &data_path, + &cell.instrument, + Some(cell.window_ms.0), + Some(cell.window_ms.1), + )?; + let eff_from = aura_ingest::epoch_ns_to_unix_ms(eff_from_ts); + let eff_to = aura_ingest::epoch_ns_to_unix_ms(eff_to_ts); + let gap_months = interior_gap_months(&months, cell.window_ms); + if eff_from == cell.window_ms.0 && eff_to == cell.window_ms.1 && gap_months.is_empty() { + return None; // fully covered — nothing to annotate + } + Some(aura_registry::CellCoverage { + effective_from_ms: eff_from, + effective_to_ms: eff_to, + gap_months, + }) + } +} + +/// The whole `(year, month)`s missing INSIDE `months`' own span (mirrors the +/// interior-gap walk `data_coverage_report`, main.rs, performs over a full +/// archive listing) that also fall inside `window_ms` (Unix-ms) — one +/// `"YYYY-MM"` entry per missing month, never a collapsed range: the +/// registry's `CellCoverage::gap_months` is a flat list an aggregate counts +/// directly. `months` is assumed sorted ([`aura_ingest::list_m1_months`]'s own +/// contract). +fn interior_gap_months(months: &[(u16, u8)], window_ms: (i64, i64)) -> Vec { + let from_ym = data_server::records::unix_ms_to_year_month(window_ms.0); + let to_ym = data_server::records::unix_ms_to_year_month(window_ms.1); + let mut out = Vec::new(); + for pair in months.windows(2) { + let (prev, next) = (pair[0], pair[1]); + let mut ym = crate::next_year_month(prev); + while ym != next { + if ym >= from_ym && ym <= to_ym { + out.push(crate::fmt_year_month(ym)); + } + ym = crate::next_year_month(ym); + } + } + out } /// Stdout shape of a campaign run. `Full` is `aura campaign run` (emit-gated @@ -428,8 +475,10 @@ pub(crate) enum RunPresentation { } /// `aura campaign run `: resolve, gate, execute, emit. Every `Err` -/// is a refusal `campaign_cmd` renders as `aura: {msg}` + exit 1. -pub(crate) fn run_campaign(target: &str, env: &Env) -> Result<(), String> { +/// is a refusal `campaign_cmd` renders as `aura: {msg}` + exit 1; an `Ok` +/// carries the failed-cell count (#272) so the caller can exit 3 on a +/// completed-with-failures run via `exit_on_campaign_result`. +pub(crate) fn run_campaign(target: &str, env: &Env) -> Result { // Project gate FIRST: nothing (not even the file-sugar registration) // touches a store outside a project. if env.provenance().is_none() { @@ -491,7 +540,7 @@ pub(crate) fn run_campaign_by_id( campaign_id: &str, env: &Env, presentation: RunPresentation, -) -> Result<(), String> { +) -> Result { let run = run_campaign_returning(campaign_id, env)?; present_campaign(run, presentation, env) } @@ -600,7 +649,7 @@ fn present_campaign( run: CampaignRun, presentation: RunPresentation, env: &Env, -) -> Result<(), String> { +) -> Result { let CampaignRun { outcome, campaign, strategies, server } = run; // Zero-survivor stderr notes (exit stays 0 — a null result is a valid @@ -621,6 +670,22 @@ fn present_campaign( } } + // #272: failed-cell notes (the gate-empty idiom's sibling) — a failed + // cell never aborts the campaign; its fault is recorded and the run + // continues over the remaining cells. + let mut failed = 0usize; + let mut fail_labels: Vec = Vec::new(); + for cell in &outcome.record.cells { + if let Some(f) = &cell.fault { + failed += 1; + fail_labels.push(format!("{}: {}", cell.instrument, cell_fault_kind_label(f.kind))); + eprintln!( + "aura: cell ({}, {}, [{}, {}]) failed at stage {}: {} — recorded, campaign continues", + cell.strategy, cell.instrument, cell.window_ms.0, cell.window_ms.1, f.stage, f.detail + ); + } + } + // Emission: emit-gated family/selection lines per cell, then the // always-on final record line. Matched by NAME against the two closed- // vocabulary terms (self-evident at the call site, unlike positional @@ -668,6 +733,16 @@ fn present_campaign( serde_json::to_string(&CampaignRunLine { campaign_run: &outcome.record }) .expect("campaign run record serializes") ); + eprintln!( + "campaign run {} recorded: {} cells{}", + outcome.record.run, + outcome.record.cells.len(), + if failed == 0 { + String::new() + } else { + format!(", {failed} failed ({})", fail_labels.join(", ")) + } + ); } // Trace persistence runs LAST, after the always-on record line (0109): @@ -685,7 +760,22 @@ fn present_campaign( env, )?; } - Ok(()) + Ok(failed) +} + +/// #272: the fault-kind label the failed-cell summary line names — the same +/// closed `CellFaultKind` vocabulary an aggregate over `campaign_runs.jsonl` +/// counts by. Debug-leak-free (a fixed lowercase-snake label, not a Debug +/// frame of the enum variant). +fn cell_fault_kind_label(kind: aura_registry::CellFaultKind) -> &'static str { + use aura_registry::CellFaultKind as K; + match kind { + K::NoData => "no_data", + K::Bind => "bind", + K::Run => "run", + K::Panic => "panic", + K::Window => "window", + } } /// Which drained wrap-convention channel a requested tap persists from on a @@ -1466,4 +1556,24 @@ mod tests { dedup.dedup(); assert_eq!(dedup.len(), knobs.len(), "knob names must be pairwise distinct: {knobs:?}"); } + + #[test] + /// #272: `interior_gap_months` names a whole calendar month missing + /// BETWEEN two present archive months (the Copper failure shape) when the + /// requested window spans across it — not just when the window's own + /// bounds fall short of full coverage. GAPSYM-shaped input (files at + /// 2024-01..02 and 2024-05..06, absent 03..04) over a Jan-through-June + /// window must name exactly the two missing interior months. + fn interior_gap_months_names_a_whole_missing_month_spanned_by_the_window() { + let months = [(2024u16, 1u8), (2024, 2), (2024, 5), (2024, 6)]; + // 2024-01-01T00:00:00Z .. 2024-07-01T00:00:00Z (Unix ms) — spans every + // present month plus the March/April hole. + let window_ms = (1_704_067_200_000i64, 1_719_792_000_000i64); + let gaps = interior_gap_months(&months, window_ms); + assert_eq!( + gaps, + vec!["2024-03".to_string(), "2024-04".to_string()], + "both interior gap months must be named, in order" + ); + } } diff --git a/crates/aura-cli/src/main.rs b/crates/aura-cli/src/main.rs index 4b982a7..65cd52f 100644 --- a/crates/aura-cli/src/main.rs +++ b/crates/aura-cli/src/main.rs @@ -2714,8 +2714,8 @@ fn parse_scalar_csv(csv: &str) -> Option> { // selects the loaded-blueprint branch. There is no second built-in grammar anymore // (#159 demo retirement / #220 axis generalization): without a blueprint the // dispatcher prints a usage error and exits 2; a blueprint with `--real` routes to -// the campaign path. Usage errors (clap parse + argv-applicability guards) exit 2; -// runtime failures exit 1. +// the campaign path. Usage errors exit 2; runtime refusals exit 1; a run that +// completes with ≥1 failed cell exits 3 (#272); a clean run exits 0. /// The single build-time commit provenance (`option_env!("AURA_COMMIT")`, /// falling back to `"unknown"`): both `--version` (via `version_string`) and @@ -3426,6 +3426,20 @@ fn exit_axis_register_error(e: AxisRegisterError) -> ! { } } +/// Terminate per a campaign-path result (#272): a String error is a refusal +/// (exit 1); an Ok carrying the failed-cell count exits 3 when any cell failed +/// ("completed with failed cells"), else returns cleanly (exit 0). +pub(crate) fn exit_on_campaign_result(r: Result) { + match r { + Err(m) => { + eprintln!("aura: {m}"); + std::process::exit(1); + } + Ok(0) => {} + Ok(_) => std::process::exit(3), + } +} + /// The shared campaign window in Unix-ms, clipped to the archive. `full_window` /// probes the ARCHIVE's actual first/last bar in range (never the literal ms /// request) and returns aura's native epoch-ns `Timestamp` (the ms->ns crossing @@ -3669,10 +3683,7 @@ fn dispatch_generalize(a: GeneralizeCmd, env: &project::Env) { // sweep + walkforward only); trace-writing is out of scope here. trace: false, }; - verb_sugar::run_generalize_sugar(&inv, &metric, env).unwrap_or_else(|m| { - eprintln!("aura: {m}"); - std::process::exit(1); - }); + exit_on_campaign_result(verb_sugar::run_generalize_sugar(&inv, &metric, env)); } fn dispatch_runs(a: RunsCmd, env: &project::Env) { @@ -3965,10 +3976,7 @@ fn dispatch_sweep(a: SweepCmd, env: &project::Env) { // `name_persist`'s bool (true iff `--trace` was given). trace: persist, }; - verb_sugar::run_sweep_sugar(&inv, env).unwrap_or_else(|m| { - eprintln!("aura: {m}"); - std::process::exit(1); - }); + exit_on_campaign_result(verb_sugar::run_sweep_sugar(&inv, env)); } DataChoice::Synthetic => { // The synthetic in-process family path is still reduce-only @@ -4069,7 +4077,7 @@ fn dispatch_walkforward(a: WalkforwardCmd, env: &project::Env) { // (unchanged since 0109 — walkforward always nominates). trace, }; - verb_sugar::run_walkforward_sugar( + let result = verb_sugar::run_walkforward_sugar( &inv, WINNER_SELECTION_METRIC, verb_sugar::WfWindows { @@ -4079,11 +4087,8 @@ fn dispatch_walkforward(a: WalkforwardCmd, env: &project::Env) { }, select, env, - ) - .unwrap_or_else(|m| { - eprintln!("aura: {m}"); - std::process::exit(1); - }); + ); + exit_on_campaign_result(result); return; } // Synthetic in-process family path — unchanged (#220 non-goal); it @@ -4184,7 +4189,7 @@ fn dispatch_mc(a: McCmd, env: &project::Env) { // is out of #224's scope here. trace: false, }; - verb_sugar::run_mc_sugar( + let result = verb_sugar::run_mc_sugar( &inv, WINNER_SELECTION_METRIC, verb_sugar::WfWindows { @@ -4194,11 +4199,8 @@ fn dispatch_mc(a: McCmd, env: &project::Env) { }, verb_sugar::McKnobs { resamples, block_len, seed }, env, - ) - .unwrap_or_else(|m| { - eprintln!("aura: {m}"); - std::process::exit(1); - }); + ); + exit_on_campaign_result(result); return; } // Synthetic seed family (unchanged, #220 non-goal): closed diff --git a/crates/aura-cli/src/research_docs.rs b/crates/aura-cli/src/research_docs.rs index 12a7b8e..520f4ab 100644 --- a/crates/aura-cli/src/research_docs.rs +++ b/crates/aura-cli/src/research_docs.rs @@ -416,7 +416,14 @@ fn campaign_runs(campaign: Option<&str>, run: Option, env: &Env) -> Resul Ok(()) } +/// #272: `Run` threads a failed-cell count (exit 3 on completed-with-failures, +/// via `exit_on_campaign_result`), so it is pulled out of the unified +/// `Result<(), String>` match the other four subcommands still share. pub fn campaign_cmd(cmd: CampaignCmd, env: &Env) { + if let CampaignSub::Run { target } = &cmd.sub { + crate::exit_on_campaign_result(crate::campaign_run::run_campaign(target, env)); + return; + } let result = match &cmd.sub { CampaignSub::Validate { file } => validate_campaign_file(file, env), CampaignSub::Introspect(i) => { @@ -424,8 +431,8 @@ pub fn campaign_cmd(cmd: CampaignCmd, env: &Env) { introspect_campaign(i) } CampaignSub::Register { file } => register_campaign(file, env), - CampaignSub::Run { target } => crate::campaign_run::run_campaign(target, env), CampaignSub::Runs { campaign, run } => campaign_runs(campaign.as_deref(), *run, env), + CampaignSub::Run { .. } => unreachable!("handled above"), }; if let Err(m) = result { eprintln!("aura: {m}"); diff --git a/crates/aura-cli/src/verb_sugar.rs b/crates/aura-cli/src/verb_sugar.rs index 9cbf3f5..b57e450 100644 --- a/crates/aura-cli/src/verb_sugar.rs +++ b/crates/aura-cli/src/verb_sugar.rs @@ -333,11 +333,13 @@ fn validate_before_register( } /// Run one dissolved sweep invocation end-to-end: register the generated -/// documents, then execute through the one campaign path in sugar mode. +/// documents, then execute through the one campaign path in sugar mode. The +/// `Ok` carries the failed-cell count (#272), propagated verbatim from the +/// one campaign executor. pub(crate) fn run_sweep_sugar( inv: &SugarInvocation, env: &crate::project::Env, -) -> Result<(), String> { +) -> Result { let generated = translate_sweep(inv)?; let probe_params = probe_params_from(inv.axes); validate_before_register(&generated.process, &generated.campaign, inv.blueprint_canonical, &probe_params, env)?; @@ -349,37 +351,36 @@ pub(crate) fn run_sweep_sugar( /// Build the `CrossInstrument` family members from an executed generalize /// campaign: each cell's nominee `RunReport`, in cell (instrument doc) order — /// the same order + shape the inline `run_generalize` built its `members` in, -/// each already carrying `manifest.instrument`. Debug-asserts one member per -/// cell: generalize's pipeline has no gate stage, so every cell nominates — -/// a cell silently missing its nominee here would be a campaign-executor -/// regression, not a legitimate generalize outcome, and the filter_map below -/// must not swallow it unnoticed. +/// each already carrying `manifest.instrument`. #272: a failed cell nominates +/// nothing, so `members.len() < outcome.cells.len()` is now a legitimate +/// outcome (the caller counts and surfaces the failed cells separately, via +/// `outcome.record.cells[].fault`) — no longer an invariant this function +/// itself enforces. fn cross_instrument_members( outcome: &aura_campaign::CampaignOutcome, ) -> Vec { - let members: Vec = outcome + outcome .cells .iter() .filter_map(|c| c.nominee.as_ref().map(|(_, report)| report.clone())) - .collect(); - debug_assert_eq!( - members.len(), - outcome.cells.len(), - "generalize has no gate stage — every cell must nominate" - ); - members + .collect() } /// Run one dissolved `generalize` invocation end-to-end: register the generated /// documents, run through the one campaign path, then reprint today's exact /// `{"generalize":{…}}` + `{"family_id":…}` lines from the recorded outcome and /// persist the `CrossInstrument` family. The stdout is byte-identical to the -/// retired welded path (the committed exact-grade anchor is the gate). +/// retired welded path (the committed exact-grade anchor is the gate). The +/// `Ok` carries the failed-cell count (#272): a failed cell nominates nothing +/// (the existing no-nominee -> `missing` path already handles it), counted +/// from the recorded cells' own `fault` field rather than from the missing +/// nominee, so the count is right even when some OTHER member of the cell's +/// pipeline is what actually failed. pub(crate) fn run_generalize_sugar( inv: &SugarInvocation, metric: &str, env: &crate::project::Env, -) -> Result<(), String> { +) -> Result { let generated = translate_generalize(inv, metric)?; let probe_params = probe_params_from(inv.axes); validate_before_register(&generated.process, &generated.campaign, inv.blueprint_canonical, &probe_params, env)?; @@ -403,7 +404,8 @@ pub(crate) fn run_generalize_sugar( .append_family(&inv.name, aura_registry::FamilyKind::CrossInstrument, &members) .map_err(|e| e.to_string())?; println!("{{\"family_id\":\"{family_id}\"}}"); - Ok(()) + let failed = run.outcome.record.cells.iter().filter(|c| c.fault.is_some()).count(); + Ok(failed) } /// The two generated documents of one dissolved `walkforward` invocation. @@ -489,14 +491,18 @@ pub(crate) fn register_generated_wf( /// documents, run through the one campaign path, then reprint the per-window member /// lines + the summary line from the recorded `WalkForward` family. The stdout /// summary line is byte-identical to the retired welded path (the committed -/// exact-grade anchor is the gate). +/// exact-grade anchor is the gate). #272: the invocation's one cell either +/// completed (`Ok(0)`) or failed (`Ok(1)`) — a failed single cell is not an +/// error, since a completed-with-failed-cell run is the campaign path's own +/// vocabulary (exit 3 at the dispatch boundary), never an "absent bootstrap" +/// refusal the retired welded path's `.ok_or` would have raised. pub(crate) fn run_walkforward_sugar( inv: &SugarInvocation, metric: &str, w: WfWindows, select: SelectRule, env: &crate::project::Env, -) -> Result<(), String> { +) -> Result { let generated = translate_walkforward(inv, metric, w, select)?; let probe_params = probe_params_from(inv.axes); validate_before_register(&generated.process, &generated.campaign, inv.blueprint_canonical, &probe_params, env)?; @@ -504,6 +510,11 @@ pub(crate) fn run_walkforward_sugar( let (_process_id, campaign_id) = register_generated_wf(®, &generated)?; let run = crate::campaign_run::run_campaign_returning(&campaign_id, env)?; + if let Some(f) = run.outcome.record.cells.iter().find_map(|c| c.fault.as_ref()) { + eprintln!("aura: walkforward cell failed at stage {}: {}", f.stage, f.detail); + return Ok(1); + } + // The single cell's walk_forward StageFamily. let wf = run .outcome @@ -549,7 +560,7 @@ pub(crate) fn run_walkforward_sugar( env, )?; } - Ok(()) + Ok(0) } /// The two generated documents of one dissolved `mc` invocation. @@ -642,13 +653,16 @@ pub(crate) fn register_generated_mc( /// byte-identical to the retired welded path (the committed exact-grade anchor is /// the gate). mc prints ONLY the one summary line — no per-window member lines /// (unlike walkforward): the monte_carlo stage is a terminal annotator, not a family. +/// #272: the invocation's one cell either completed (`Ok(0)`) or failed +/// (`Ok(1)`) — a failed single cell is not an error (see `run_walkforward_sugar`'s +/// twin note). pub(crate) fn run_mc_sugar( inv: &SugarInvocation, metric: &str, w: WfWindows, mc: McKnobs, env: &crate::project::Env, -) -> Result<(), String> { +) -> Result { let generated = translate_mc(inv, metric, w, mc)?; let probe_params = probe_params_from(inv.axes); validate_before_register(&generated.process, &generated.campaign, inv.blueprint_canonical, &probe_params, env)?; @@ -656,6 +670,11 @@ pub(crate) fn run_mc_sugar( let (_process_id, campaign_id) = register_generated_mc(®, &generated)?; let run = crate::campaign_run::run_campaign_returning(&campaign_id, env)?; + if let Some(f) = run.outcome.record.cells.iter().find_map(|c| c.fault.as_ref()) { + eprintln!("aura: mc cell failed at stage {}: {}", f.stage, f.detail); + return Ok(1); + } + // The single cell's terminal monte_carlo StageRealization carries the pooled-OOS // bootstrap (record path: outcome.record.cells[].stages[].bootstrap). let boot = run @@ -670,7 +689,7 @@ pub(crate) fn run_mc_sugar( }) .ok_or("mc produced no pooled-OOS bootstrap")?; println!("{}", crate::mc_r_bootstrap_json(boot)); - Ok(()) + Ok(0) } #[cfg(test)] @@ -1140,4 +1159,73 @@ mod tests { assert_eq!(stored_process, process_to_json(&generated.process)); let _ = std::fs::remove_dir_all(&dir); } + + /// A minimal `RunReport` stamped with `instrument`, distinguishable in a + /// members list by nothing but that stamp — the field + /// `cross_instrument_members` preserves verbatim from a nominating cell. + fn fake_report(instrument: &str) -> aura_engine::RunReport { + aura_engine::RunReport { + manifest: aura_engine::RunManifest { + commit: String::new(), + params: vec![], + defaults: vec![], + window: (aura_engine::Timestamp(0), aura_engine::Timestamp(1)), + seed: 0, + broker: "test".to_string(), + selection: None, + instrument: Some(instrument.to_string()), + topology_hash: None, + project: None, + }, + metrics: aura_engine::RunMetrics { + total_pips: 0.0, + max_drawdown: 0.0, + bias_sign_flips: 0, + r: None, + }, + } + } + + fn cell_outcome(nominee_instrument: Option<&str>) -> aura_campaign::CellOutcome { + aura_campaign::CellOutcome { + families: vec![], + selections: vec![], + nominee: nominee_instrument.map(|i| (vec![], fake_report(i))), + } + } + + /// #272: `cross_instrument_members` skips a cell with no nominee (a + /// failed cell nominates nothing) rather than padding the members list or + /// aborting — the surviving cells' reports still reach the persisted + /// `CrossInstrument` family, in cell order, with the failed cell simply + /// absent (not a placeholder). Three cells (AAA nominates, BBB fails, CCC + /// nominates) must yield exactly the two survivors' reports, in order. + #[test] + fn cross_instrument_members_skips_a_failed_cell_and_keeps_survivor_order() { + let outcome = aura_campaign::CampaignOutcome { + record: aura_registry::CampaignRunRecord { + campaign: String::new(), + process: String::new(), + run: 0, + seed: 0, + cells: vec![], + generalizations: vec![], + trace_name: None, + }, + run: 0, + cells: vec![ + cell_outcome(Some("AAA")), + cell_outcome(None), + cell_outcome(Some("CCC")), + ], + }; + let members = cross_instrument_members(&outcome); + let instruments: Vec> = + members.iter().map(|r| r.manifest.instrument.clone()).collect(); + assert_eq!( + instruments, + vec![Some("AAA".to_string()), Some("CCC".to_string())], + "the failed (no-nominee) cell is skipped, the two survivors keep their order" + ); + } } diff --git a/crates/aura-cli/tests/project_sweep_campaign.rs b/crates/aura-cli/tests/project_sweep_campaign.rs index 35d848d..a7d1dcd 100644 --- a/crates/aura-cli/tests/project_sweep_campaign.rs +++ b/crates/aura-cli/tests/project_sweep_campaign.rs @@ -331,7 +331,7 @@ fn project_node_open_param_runs_one_campaign_cell() { // Archive-gated: skip cleanly on a data-less machine (the member-data // refusal, never a panic) — mirroring campaign_run_real_e2e_*. - if run.status.code() == Some(1) + if run.status.code() == Some(3) && (out.contains("no recorded geometry") || out.contains("no data for instrument")) { eprintln!("skip: no local GER40 data for the campaign cell"); diff --git a/crates/aura-cli/tests/research_docs.rs b/crates/aura-cli/tests/research_docs.rs index cb4c9dd..49f5ed1 100644 --- a/crates/aura-cli/tests/research_docs.rs +++ b/crates/aura-cli/tests/research_docs.rs @@ -5,7 +5,10 @@ use std::path::{Path, PathBuf}; mod common; -use common::{ScratchGuard, ScratchPath, fresh_project, fresh_project_with_data}; +use common::{ + ScratchGuard, ScratchPath, fresh_project, fresh_project_with_data, + fresh_project_with_gapped_data, +}; /// A fresh, unique working directory for a process test that persists /// content-addressed documents under `./runs/` (so `aura process register` @@ -1999,7 +2002,7 @@ fn campaign_run_refuses_zero_resamples_monte_carlo_before_any_member_runs() { /// parse-valid gate (`parse_valid_campaign` — the same `validate_campaign` + /// `doc_fault_prose` composition `campaign validate` shares), before /// registration, before the referential tier, before any member runs — and -/// therefore before the (Task-3-doomed) loud-deferral note, whose absence is +/// therefore before the retired loud-deferral note, whose absence is /// pinned here. #[test] fn campaign_run_refuses_unknown_tap_at_validate() { @@ -2037,14 +2040,13 @@ fn campaign_run_refuses_unknown_tap_at_validate() { /// 0109 split (b) — the control: an in-vocabulary tap ("equity") sails /// through the new intrinsic gate (no UnknownTap prose) and the run proceeds -/// to refuse at the member-data seam, exactly as a tap-less run over the -/// [1, 2] 1970 epoch-ms window does (data-less host: missing geometry; -/// data-ful host: no archive file overlaps) — exit 1 either way, and no -/// trace output (tracing never starts on a run that dies before members; -/// there is no "traces persisted" summary line). NOTE: the 0107 -/// loud-deferral eprintln still fires in this task's tree and is -/// deliberately NOT asserted either way here — Task 3 deletes it and -/// tightens this test to pin its absence. +/// to the member-data seam, exactly as a tap-less run over the [1, 2] 1970 +/// epoch-ms window does (data-less host: missing geometry; data-ful host: no +/// archive file overlaps) — #272: the member-data refusal is now a CONTAINED +/// cell fault, not a campaign abort, so the run completes (exit 3, "completed +/// with failed cells") and the always-on record line still carries the +/// failed cell's fault. The retired 0107 loud-deferral note ("persist_taps not +/// yet honored") must not appear — its absence is asserted below. #[test] fn campaign_run_valid_tap_reaches_the_member_data_seam() { let (dir, _fixture) = fresh_project(); @@ -2063,7 +2065,7 @@ fn campaign_run_valid_tap_reaches_the_member_data_seam() { &campaign_doc_json(&bp_id, &proc_id, (1, 2), "\"equity\"", ""), ); let (out, code) = run_code_in(&dir, &["campaign", "run", "taps.campaign.json"]); - assert_eq!(code, Some(1), "stdout/stderr: {out}"); + assert_eq!(code, Some(3), "stdout/stderr: {out}"); assert!( !out.contains("unknown tap"), "\"equity\" is in the closed vocabulary and must pass validate: {out}" @@ -2072,17 +2074,25 @@ fn campaign_run_valid_tap_reaches_the_member_data_seam() { out.contains("no recorded geometry") || out.contains("no data for instrument"), "the refusal names the data condition: {out}" ); - assert!( - !out.contains("traces persisted"), - "no trace summary on a run that never reached a member: {out}" - ); assert!( !out.contains("persist_taps not yet honored"), "the 0107 deferral line is retired; persist_taps is honored from 0109 on: {out}" ); + // The record shows the cell as failed and the summary names the count — + // not the brittle "no traces persisted" absence: a contained fault still + // reaches (and passes through) the trace-persistence tail, which honestly + // reports 0 cells written, so "traces persisted:" DOES appear now. assert!( - !out.contains("traces persisted:"), - "a member-data refusal aborts inside execute, before any tracing: {out}" + out.contains("1 failed"), + "the campaign-run summary names the one failed cell: {out}" + ); + let record_line = out + .lines() + .find(|l| l.starts_with("{\"campaign_run\":")) + .expect("the always-on final campaign_run line"); + assert!( + record_line.contains("\"fault\""), + "the recorded cell carries its fault: {record_line}" ); } @@ -2127,7 +2137,7 @@ fn campaign_run_real_e2e_sweep_gate_walkforward() { let (out, code) = run_code_in(&dir, &["campaign", "run", "wf.campaign.json"]); // Skip on a data-less machine: the member-data refusal, never a panic. - if code == Some(1) + if code == Some(3) && (out.contains("no recorded geometry") || out.contains("no data for instrument")) { eprintln!("skip: no local GER40 data for the campaign e2e"); @@ -2627,7 +2637,7 @@ fn campaign_run_real_e2e_bindings_override_rebinds_price_to_open() { write_doc(&dir, "open.campaign.json", &rebound); let (out_close, code_close) = run_code_in(&dir, &["campaign", "run", "close.campaign.json"]); - if code_close == Some(1) + if code_close == Some(3) && (out_close.contains("no recorded geometry") || out_close.contains("no data for instrument")) { eprintln!("skip: no local GER40 data for the binding-override e2e"); @@ -2691,7 +2701,7 @@ fn campaign_run_real_e2e_non_default_regime_stamps_its_own_stop() { write_doc(&dir, "regime.campaign.json", &with_regime); let (out, code) = run_code_in(&dir, &["campaign", "run", "regime.campaign.json"]); - if code == Some(1) + if code == Some(3) && (out.contains("no recorded geometry") || out.contains("no data for instrument")) { eprintln!("skip: no local GER40 data for the campaign regime e2e"); @@ -2761,7 +2771,7 @@ fn campaign_run_real_e2e_two_regimes_each_stamp_their_own_stop() { write_doc(&dir, "tworegimes.campaign.json", &with_regimes); let (out, code) = run_code_in(&dir, &["campaign", "run", "tworegimes.campaign.json"]); - if code == Some(1) + if code == Some(3) && (out.contains("no recorded geometry") || out.contains("no data for instrument")) { eprintln!("skip: no local GER40 data for the campaign two-regime e2e"); @@ -2849,7 +2859,7 @@ fn campaign_persist_non_default_regime_does_not_false_fail_c1() { let (out, code) = run_code_in(&dir, &["campaign", "run", "regime-persist.campaign.json"]); // Skip on a data-less machine: the member-data refusal, never a panic. - if code == Some(1) + if code == Some(3) && (out.contains("no recorded geometry") || out.contains("no data for instrument")) { eprintln!("skip: no local GER40 data for the campaign regime-persist e2e"); @@ -2929,7 +2939,7 @@ fn campaign_run_real_e2e_costed_campaign_persists_without_drift_alarm() { let (out, code) = run_code_in(&dir, &["campaign", "run", "cost-persist.campaign.json"]); // Skip on a data-less machine: the member-data refusal, never a panic. - if code == Some(1) + if code == Some(3) && (out.contains("no recorded geometry") || out.contains("no data for instrument")) { eprintln!("skip: no local GER40 data for the campaign cost-persist e2e"); @@ -3024,7 +3034,7 @@ fn campaign_run_real_e2e_net_r_equity_tap_persists_the_cost_adjusted_curve() { let (out, code) = run_code_in(&dir, &["campaign", "run", "net-persist.campaign.json"]); // Skip on a data-less machine: the member-data refusal, never a panic. - if code == Some(1) + if code == Some(3) && (out.contains("no recorded geometry") || out.contains("no data for instrument")) { eprintln!("skip: no local GER40 data for the campaign net-persist e2e"); @@ -3138,7 +3148,7 @@ fn campaign_run_real_e2e_vol_slippage_cost_persists_without_drift_alarm() { let (out, code) = run_code_in(&dir, &["campaign", "run", "volcost-persist.campaign.json"]); // Skip on a data-less machine: the member-data refusal, never a panic. - if code == Some(1) + if code == Some(3) && (out.contains("no recorded geometry") || out.contains("no data for instrument")) { eprintln!("skip: no local GER40 data for the campaign vol-slippage-persist e2e"); @@ -3227,7 +3237,7 @@ fn campaign_persist_two_regimes_land_in_distinct_trace_dirs() { let (out, code) = run_code_in(&dir, &["campaign", "run", "tworegimes-persist.campaign.json"]); // Skip on a data-less machine: the member-data refusal, never a panic. - if code == Some(1) + if code == Some(3) && (out.contains("no recorded geometry") || out.contains("no data for instrument")) { eprintln!("skip: no local GER40 data for the two-regime persist e2e"); @@ -3305,7 +3315,7 @@ fn campaign_run_real_e2e_skips_unproducible_tap_gracefully() { let (out, code) = run_code_in(&dir, &["campaign", "run", "mixedtap.campaign.json"]); // Skip on a data-less machine: the member-data refusal, never a panic. - if code == Some(1) + if code == Some(3) && (out.contains("no recorded geometry") || out.contains("no data for instrument")) { eprintln!("skip: no local GER40 data for the campaign e2e"); @@ -3419,7 +3429,7 @@ fn campaign_run_real_e2e_cost_block_nets_members_and_persists_net_r_equity() { // (a) the cost-less baseline: net == gross, and net_r_equity skips with // the remedy notice while equity still persists. let (out_gross, code_gross) = run_code_in(&dir, &["campaign", "run", "costless.campaign.json"]); - if code_gross == Some(1) + if code_gross == Some(3) && (out_gross.contains("no recorded geometry") || out_gross.contains("no data for instrument")) { eprintln!("skip: no local GER40 data for the cost e2e"); @@ -3542,7 +3552,7 @@ fn campaign_run_real_e2e_carry_cost_persists_without_drift_alarm() { let (out, code) = run_code_in(&dir, &["campaign", "run", "carrycost-persist.campaign.json"]); // Skip on a data-less machine: the member-data refusal, never a panic. - if code == Some(1) + if code == Some(3) && (out.contains("no recorded geometry") || out.contains("no data for instrument")) { eprintln!("skip: no local GER40 data for the campaign carry-cost-persist e2e"); @@ -3614,7 +3624,7 @@ fn campaign_run_real_e2e_sweep_monte_carlo_per_survivor() { let (out, code) = run_code_in(&dir, &["campaign", "run", "mc.campaign.json"]); // Skip on a data-less machine: the member-data refusal, never a panic. - if code == Some(1) + if code == Some(3) && (out.contains("no recorded geometry") || out.contains("no data for instrument")) { eprintln!("skip: no local GER40 data for the campaign e2e"); @@ -3662,10 +3672,12 @@ fn campaign_run_real_e2e_sweep_monte_carlo_per_survivor() { /// by FILE and once by its own resulting content ID must refuse with the /// byte-identical prose — proof the two addressing modes are not two /// independently drifting code paths. No real data needed: with the [1, 2] -/// 1970 window, both runs pass the (now mc-permitting) preflight and refuse -/// at the member-data seam — no recorded geometry on a data-less host, no -/// overlapping archive window on a data-ful one — exit 1 and byte-identical -/// prose either way (the message depends only on env/instrument/window). +/// 1970 window, both runs pass the (now mc-permitting) preflight and the one +/// cell fails at the member-data seam — no recorded geometry on a data-less +/// host, no overlapping archive window on a data-ful one — #272: the failed +/// cell is contained, so the campaign run still completes (exit 3, "completed +/// with failed cells") and byte-identical prose either way (the message +/// depends only on env/instrument/window). #[test] fn campaign_run_by_content_id_matches_file_sugar_refusal() { let (dir, _fixture) = fresh_project(); @@ -3681,7 +3693,7 @@ fn campaign_run_by_content_id_matches_file_sugar_refusal() { write_doc(&dir, "mc2.campaign.json", &campaign_doc_json(&bp_id, &proc_id, (1, 2), "", "")); let (file_out, file_code) = run_code_in(&dir, &["campaign", "run", "mc2.campaign.json"]); - assert_eq!(file_code, Some(1), "stdout/stderr: {file_out}"); + assert_eq!(file_code, Some(3), "stdout/stderr: {file_out}"); let (reg_out, reg_code) = run_code_in(&dir, &["campaign", "register", "mc2.campaign.json"]); assert_eq!(reg_code, Some(0), "register failed: {reg_out}"); @@ -3696,7 +3708,7 @@ fn campaign_run_by_content_id_matches_file_sugar_refusal() { .trim_start_matches("content:") .to_string(); let (id_out, id_code) = run_code_in(&dir, &["campaign", "run", &id]); - assert_eq!(id_code, Some(1), "stdout/stderr: {id_out}"); + assert_eq!(id_code, Some(3), "stdout/stderr: {id_out}"); let file_line = file_out.lines().find(|l| l.starts_with("aura: ")).expect("file refusal line"); let id_line = id_out.lines().find(|l| l.starts_with("aura: ")).expect("id refusal line"); @@ -3797,7 +3809,7 @@ fn campaign_run_accepts_a_graph_register_seeded_strategy() { let proc_id = register_process_doc(&dir, "onramp.process.json", SWEEP_ONLY_PROCESS_DOC); write_doc(&dir, "onramp.campaign.json", &campaign_doc_json(&bp_id, &proc_id, (1, 2), "", "")); let (out, code) = run_code_in(&dir, &["campaign", "run", "onramp.campaign.json"]); - assert_eq!(code, Some(1), "stdout/stderr: {out}"); + assert_eq!(code, Some(3), "stdout/stderr: {out}"); assert!( out.contains("no recorded geometry") || out.contains("no data for instrument"), "referencing + axis-binding must succeed, reaching the member-data \ @@ -3841,3 +3853,76 @@ fn campaign_run_invalid_file_refuses_before_touching_store() { "an invalid document must not create a store entry" ); } + +/// #272: a two-window campaign over GAPSYM where one window falls inside the +/// March-April hole records that cell as failed (no_data) while the covered +/// window's cell completes and persists — exit 3, both cells in one run record. +#[test] +fn campaign_over_a_gapped_archive_records_the_uncovered_cell_and_continues() { + let (dir, _fixture) = fresh_project_with_gapped_data(); + let runs_dir = dir.join("runs"); + std::fs::remove_dir_all(&runs_dir).ok(); + let _cleanup = ScratchGuard(vec![ + ScratchPath::Dir(runs_dir.clone()), + ScratchPath::File(dir.join("gap.process.json")), + ScratchPath::File(dir.join("gap.campaign.json")), + ]); + let bp_id = seed_blueprint(&dir, "campaign-run-gap-seed"); + let proc_id = register_process_doc(&dir, "gap.process.json", SWEEP_ONLY_PROCESS_DOC); + // Window 0 (2024-01, Unix-ms [1704067200000, 1706745600000)) is fully + // archived by `fresh_project_with_gapped_data` (GAPSYM 2024-01..02); + // window 1 (2024-03, [1709251200000, 1711929600000)) falls entirely + // inside the deliberate March-April hole. + let doc = format!( + r#"{{ + "format_version": 1, + "kind": "campaign", + "name": "gap-cells", + "data": {{ "instruments": ["GAPSYM"], + "windows": [ {{ "from_ms": 1704067200000, "to_ms": 1706745600000 }}, + {{ "from_ms": 1709251200000, "to_ms": 1711929600000 }} ] }}, + "strategies": [ {{ "ref": {{ "content_id": "{bp_id}" }}, + "axes": {{ "fast.length": {{ "kind": "I64", "values": [2, 4] }}, + "slow.length": {{ "kind": "I64", "values": [8, 16] }} }} }} ], + "process": {{ "ref": {{ "content_id": "{proc_id}" }} }}, + "seed": 7, + "presentation": {{ "persist_taps": [], "emit": [] }} +}}"#, + ); + write_doc(&dir, "gap.campaign.json", &doc); + let (out, code) = run_code_in(&dir, &["campaign", "run", "gap.campaign.json"]); + assert_eq!(code, Some(3), "stdout/stderr: {out}"); + + let record_line = out + .lines() + .find(|l| l.starts_with("{\"campaign_run\":")) + .expect("the always-on final campaign_run line"); + let v: serde_json::Value = + serde_json::from_str(record_line).expect("campaign_run line parses as JSON"); + let cells = v["campaign_run"]["cells"].as_array().expect("cells array"); + assert_eq!(cells.len(), 2, "one cell per window, both in one run record: {record_line}"); + + let jan_cell = &cells[0]; + let mar_cell = &cells[1]; + assert!(jan_cell["fault"].is_null(), "the January cell completes: {record_line}"); + assert!( + !jan_cell["stages"].as_array().expect("stages").is_empty(), + "the January cell realizes a walk-through stage set: {record_line}" + ); + assert!( + jan_cell["coverage"].is_object(), + "the covered window's cell is still only trading-hours-covered inside \ + a calendar-month request, so it carries a coverage annotation: {record_line}" + ); + + assert_eq!( + mar_cell["fault"]["kind"].as_str(), + Some("no_data"), + "the March cell (entirely inside the gap) fails as no_data: {record_line}" + ); + + assert!( + out.contains("1 failed"), + "the campaign-run summary names the one failed cell: {out}" + ); +} diff --git a/crates/aura-registry/src/lib.rs b/crates/aura-registry/src/lib.rs index eb9dbaf..cd308df 100644 --- a/crates/aura-registry/src/lib.rs +++ b/crates/aura-registry/src/lib.rs @@ -30,8 +30,9 @@ mod compat; mod lineage; pub use lineage::{ derive_trace_name, group_families, mc_member_reports, sweep_member_reports, - walkforward_member_reports, CampaignGeneralization, CampaignRunRecord, CellRealization, - Family, FamilyKind, FamilyRunRecord, StageBootstrap, StageRealization, StageSelection, + walkforward_member_reports, CampaignGeneralization, CampaignRunRecord, CellCoverage, + CellFault, CellFaultKind, CellRealization, Family, FamilyKind, FamilyRunRecord, + StageBootstrap, StageRealization, StageSelection, WindowFault, }; mod trace_store; @@ -1963,6 +1964,8 @@ mod tests { window_ms: (1_136_073_600_000, 1_154_390_400_000), regime: None, regime_ordinal: 0, + fault: None, + coverage: None, stages: vec![ StageRealization { block: "std::sweep".to_string(), @@ -1989,6 +1992,7 @@ mod tests { }, }), bootstrap: None, + window_faults: vec![], }, StageRealization { block: "std::gate".to_string(), @@ -1996,6 +2000,7 @@ mod tests { survivor_ordinals: Some(vec![0, 3, 4, 7]), selection: None, bootstrap: None, + window_faults: vec![], }, ], }], @@ -2029,6 +2034,8 @@ mod tests { window_ms: (0, 1000), regime: None, regime_ordinal: 0, + fault: None, + coverage: None, stages: vec![StageRealization { block: "std::monte_carlo".to_string(), family_id: None, @@ -2038,6 +2045,7 @@ mod tests { (0, per_a), (2, per_b), ])), + window_faults: vec![], }], }, CellRealization { @@ -2046,12 +2054,15 @@ mod tests { window_ms: (0, 1000), regime: None, regime_ordinal: 0, + fault: None, + coverage: None, stages: vec![StageRealization { block: "std::monte_carlo".to_string(), family_id: None, survivor_ordinals: None, selection: None, bootstrap: Some(StageBootstrap::PooledOos(pooled)), + window_faults: vec![], }], }, ], @@ -2113,6 +2124,62 @@ mod tests { ); } + /// #272: a pre-fault campaign_runs line (no fault/coverage/window_faults + /// keys) parses and re-serializes byte-identical — the additive-widening + /// guarantee for the new per-cell-fault fields. + #[test] + fn campaign_run_line_without_fault_fields_round_trips() { + let line = r#"{"campaign":"c","process":"p","run":0,"seed":7,"cells":[{"strategy":"s","instrument":"EURUSD","window_ms":[0,10],"stages":[{"block":"std::sweep","family_id":"f-0"}]}]}"#; + let rec: CampaignRunRecord = serde_json::from_str(line).expect("parse pre-fault line"); + assert_eq!(rec.cells[0].fault, None); + assert_eq!(rec.cells[0].coverage, None); + assert!(rec.cells[0].stages[0].window_faults.is_empty()); + assert_eq!(serde_json::to_string(&rec).expect("re-serialize"), line); + } + + /// #272: a cell that failed serializes its fault, and a fold-fault list + /// serializes on the stage — both absent-by-default, present when set. + #[test] + fn cell_fault_and_window_faults_serialize_when_present() { + let fault = CellFault { stage: 0, kind: CellFaultKind::NoData, detail: "no data".into() }; + assert_eq!( + serde_json::to_string(&fault).expect("fault"), + r#"{"stage":0,"kind":"no_data","detail":"no data"}"# + ); + let wf = WindowFault { window_ordinal: 2, kind: CellFaultKind::Run, detail: "boom".into() }; + assert_eq!( + serde_json::to_string(&wf).expect("window fault"), + r#"{"window_ordinal":2,"kind":"run","detail":"boom"}"# + ); + } + + /// #272: a `CellCoverage` with gap months serializes them, and its + /// effective-bounds round-trip (parse-then-reserialize) is byte-identical + /// — pinning both the populated `gap_months` shape and the + /// `skip_serializing_if = "Vec::is_empty"` omission when there are none. + #[test] + fn cell_coverage_serializes_gap_months_and_round_trips() { + let with_gaps = CellCoverage { + effective_from_ms: 0, + effective_to_ms: 1000, + gap_months: vec!["2024-02".into()], + }; + let json = serde_json::to_string(&with_gaps).expect("coverage with gaps"); + assert_eq!( + json, + r#"{"effective_from_ms":0,"effective_to_ms":1000,"gap_months":["2024-02"]}"# + ); + let parsed: CellCoverage = serde_json::from_str(&json).expect("parse coverage"); + assert_eq!(parsed, with_gaps); + + let no_gaps = + CellCoverage { effective_from_ms: 0, effective_to_ms: 1000, gap_months: vec![] }; + assert_eq!( + serde_json::to_string(&no_gaps).expect("coverage without gaps"), + r#"{"effective_from_ms":0,"effective_to_ms":1000}"# + ); + } + /// Property (#210 T3, C14/C23 widening convention): a stored /// `campaign_runs.jsonl` line whose `generalizations` entries predate the /// risk-regime axis — carrying `strategy_ordinal`/`window_ordinal`/ diff --git a/crates/aura-registry/src/lineage.rs b/crates/aura-registry/src/lineage.rs index d51469a..ec97c24 100644 --- a/crates/aura-registry/src/lineage.rs +++ b/crates/aura-registry/src/lineage.rs @@ -98,6 +98,48 @@ pub struct CampaignRunRecord { pub trace_name: Option, } +/// The recorded fault of a failed cell (#272): the pipeline stage it fired in, +/// a closed kind tag aggregations count on, and the prose detail. `no_data` is +/// one kind among others, not a special path. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct CellFault { + pub stage: usize, + pub kind: CellFaultKind, + pub detail: String, +} + +/// Closed fault-kind vocabulary (#272). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CellFaultKind { + NoData, + Bind, + Run, + Panic, + Window, +} + +/// One failed walk-forward fold (#272): the roll ordinal that produced no OOS +/// report, with the fault that killed it. The surviving folds still pool; an +/// aggregate over this stage must name the ratio. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct WindowFault { + pub window_ordinal: usize, + pub kind: CellFaultKind, + pub detail: String, +} + +/// A cell whose requested window is only partially covered by the archive +/// (#272): the effective evaluated bounds and any whole months missing inside +/// them. Recorded only when it deviates from the requested window. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct CellCoverage { + pub effective_from_ms: i64, + pub effective_to_ms: i64, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub gap_months: Vec, +} + /// One realized (strategy, instrument, window) cell: the pipeline prefix that /// actually ran (`stages` stops after an empty gate). #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -125,6 +167,13 @@ pub struct CellRealization { /// same pre-#219 byte-identical round-trip reason. #[serde(default, skip_serializing_if = "is_zero_ordinal")] pub regime_ordinal: usize, + /// Set when the cell failed (#272): the pipeline prefix in `stages` ran, + /// the faulting stage did not complete, no nominee was produced. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fault: Option, + /// Set when the archive only partially covers the requested window (#272). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub coverage: Option, } /// `CellRealization::regime_ordinal`'s serialize-skip predicate: the default @@ -151,6 +200,10 @@ pub struct StageRealization { /// serde-default widening keeps existing stored lines parseable. #[serde(default, skip_serializing_if = "Option::is_none")] pub bootstrap: Option, + /// std::walk_forward stages only (#272): folds that produced no OOS report; + /// the surviving folds pooled, and aggregates must name the ratio. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub window_faults: Vec, } /// The recorded winner of a selection-bearing stage: its ordinal in the stage's diff --git a/crates/aura-registry/tests/campaign_run_store_e2e.rs b/crates/aura-registry/tests/campaign_run_store_e2e.rs index 6752f27..100c26e 100644 --- a/crates/aura-registry/tests/campaign_run_store_e2e.rs +++ b/crates/aura-registry/tests/campaign_run_store_e2e.rs @@ -55,6 +55,8 @@ fn campaign_run(campaign: &str) -> CampaignRunRecord { window_ms: (0, 1000), regime: None, regime_ordinal: 0, + fault: None, + coverage: None, stages: vec![StageRealization { block: "std::sweep".to_string(), family_id: Some(format!("{campaign}-0-EURUSD-w0-s0-0")), @@ -77,6 +79,7 @@ fn campaign_run(campaign: &str) -> CampaignRunRecord { }, }), bootstrap: None, + window_faults: vec![], }], }], generalizations: vec![], diff --git a/docs/authoring-guide.md b/docs/authoring-guide.md index 5f32815..74c09fb 100644 --- a/docs/authoring-guide.md +++ b/docs/authoring-guide.md @@ -604,3 +604,9 @@ family charted the same way, by the handle the run prints (members keyed uniquely names one recorded run (`aura chart ` resolves it against the stored campaign documents; a name reused across runs refuses rather than guessing which one you mean). + +Exit codes: a clean run exits 0; a usage error exits 2; a refusal before any +cell runs (invalid document, missing project, unresolvable strategy) exits 1; +a run that **completes with one or more failed cells** exits 3 — the run record +and every healthy cell persist, and the failed cells are named on stderr +(#272). diff --git a/docs/design/INDEX.md b/docs/design/INDEX.md index fdd25e8..db8a4b1 100644 --- a/docs/design/INDEX.md +++ b/docs/design/INDEX.md @@ -1605,6 +1605,21 @@ the retired deferral: per-cell no-nominee skip, per-run unproducible-tap skip pure name derivation). Noted debt: `aura chart` over the campaign family ROOT (cells spanning instruments) is untested/semantically undefined — only per-cell read-back is pinned. +**Realization (#272, 2026-07-14 — per-cell fault isolation).** A member +fault (no-data, bind, run, or a caught panic) is a recorded per-cell outcome, +never a global abort: `run_cell` returns a fault-annotated `CellRealization` +(`fault: Option`, closed `CellFaultKind`) instead of `Err`, so +`execute`'s existing accumulate-then-append-once tail persists every healthy +cell and the one run record. Containment granularity is the cell for a sweep +stage (a grid hole compromises selection) and the fold for walk_forward +(surviving folds pool; failed folds recorded as `StageRealization. +window_faults`, the summary naming the ratio). `ExecFault::Registry` and +doc-shape preflight faults stay global. The CLI declares holes (per-cell +notes + a completion summary) and a run with ≥1 failed cell exits **3** +("completed with failed cells" — distinct from 0/1/2). A partially-covered +window carries a `CellCoverage` annotation (effective bounds + interior gap +months, from the #264 archive primitives). Generalize already treats a +no-nominee cell as `missing`, so a failed cell surfaces there unchanged. **Guarantee.** Construction is a distinct phase, recursive at every level. Each node type has a **factory** `params → sized concrete node` (e.g. `SMA(length)` sizes its ring buffer). A **blueprint** is the param-generic, input-role-generic