feat(campaign): aura-campaign crate — types, member_metric, preflight (0107 tasks 4-5)
New leaf library crate: the campaign-execution semantics home (#198 home decision — reusable beyond the CLI; NOT C21's project-side World). CellSpec + the MemberRunner seam (the only thing a consumer implements), MemberFault/ExecFault (Display-free, prose at the binary seam), member_metric over the 14 per-member scalars (third roster site, refs #190 — drift fails safe as a preflight refusal), and preflight: v1 pipeline shape std::sweep [std::gate]* [std::walk_forward], rankable-metric and gate-metric rosters, plateau-in-wf and deflate+plateau refusals, i64-fit guard on wf lengths — all before any member runs. Post-loop: the held quality finding (untested i64-fit arm) verified real and closed by hand — preflight_refuses_wf_length_exceeding_i64. Gates: aura-campaign 14/0, clippy -D warnings clean. refs #198
This commit is contained in:
Generated
+13
@@ -105,6 +105,19 @@ dependencies = [
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aura-campaign"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"aura-analysis",
|
||||
"aura-core",
|
||||
"aura-engine",
|
||||
"aura-registry",
|
||||
"aura-research",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aura-cli"
|
||||
version = "0.1.0"
|
||||
|
||||
@@ -18,6 +18,7 @@ members = [
|
||||
"crates/aura-ingest",
|
||||
"crates/aura-registry",
|
||||
"crates/aura-research",
|
||||
"crates/aura-campaign",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
[package]
|
||||
name = "aura-campaign"
|
||||
edition.workspace = true
|
||||
version.workspace = true
|
||||
license.workspace = true
|
||||
publish.workspace = true
|
||||
|
||||
# Deliberately NO aura-ingest / aura-std / aura-composites: harness and data
|
||||
# binding enter through the one-method MemberRunner seam (src/lib.rs), so the
|
||||
# deploy-condemned CLI scaffolding never becomes a library dep and a
|
||||
# differently-binding consumer (playground, tests) implements the same seam.
|
||||
[dependencies]
|
||||
# the scalar vocabulary: params cross the MemberRunner seam as (name, Scalar) pairs.
|
||||
aura-core = { path = "../aura-core" }
|
||||
# RunReport (the member result type) + the sweep/walk_forward orchestration machinery.
|
||||
aura-engine = { path = "../aura-engine" }
|
||||
# FamilySelection — the selection-provenance type stage selections carry.
|
||||
aura-analysis = { path = "../aura-analysis" }
|
||||
# family + campaign-run stores and the optimize/optimize_plateau/optimize_deflated selectors.
|
||||
aura-registry = { path = "../aura-registry" }
|
||||
# the campaign/process document types this crate executes.
|
||||
aura-research = { path = "../aura-research" }
|
||||
# realization payloads derive serde (the registry per-case policy, INDEX.md).
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
@@ -0,0 +1,585 @@
|
||||
//! aura-campaign — the campaign-execution library (#198, cycle 0107).
|
||||
//!
|
||||
//! Campaign *semantics* as a reusable leaf crate: cell enumeration over a
|
||||
//! campaign document's (strategy, instrument, window) matrix, preflight of
|
||||
//! the v1 executable pipeline shape, per-member gate evaluation, winner
|
||||
//! selection, and realization assembly over the registry's family machinery.
|
||||
//! Harness construction and data binding enter exclusively through the
|
||||
//! one-method [`MemberRunner`] seam, so every consumer (the CLI today; the
|
||||
//! playground and tests tomorrow) binds its own runner while the execution
|
||||
//! semantics live here once — this crate is NOT the World (C12/C21): it
|
||||
//! realizes one campaign document; it owns no topology, no data sources,
|
||||
//! and no UI.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aura_core::Scalar;
|
||||
use aura_engine::RunReport;
|
||||
use aura_registry::RegistryError;
|
||||
use aura_research::{Axis, CampaignDoc, Cmp, ProcessDoc, SelectRule, StageBlock};
|
||||
|
||||
/// One structural cell of the campaign matrix: (strategy, instrument,
|
||||
/// window) — #198 decision 7.
|
||||
pub struct CellSpec {
|
||||
pub strategy_ordinal: usize,
|
||||
/// Resolved blueprint content id (== the topology hash).
|
||||
pub strategy_id: String,
|
||||
/// Canonical blueprint bytes from the store.
|
||||
pub blueprint_json: String,
|
||||
/// The campaign's tuning axes (raw `param_space()` names).
|
||||
pub axes: BTreeMap<String, Axis>,
|
||||
pub instrument: String,
|
||||
/// Inclusive epoch-ms bounds.
|
||||
pub window_ms: (i64, i64),
|
||||
}
|
||||
|
||||
/// The harness/data binding seam — the ONLY thing a consumer implements.
|
||||
/// Params arrive as (raw axis name, value) pairs; the implementation binds
|
||||
/// them to its harness convention and runs the member over `cell.instrument`
|
||||
/// restricted to `window_ms` (inclusive epoch-ms, a sub-range of
|
||||
/// `cell.window_ms` — a walk-forward stage passes sub-windows).
|
||||
pub trait MemberRunner: Sync {
|
||||
fn run_member(
|
||||
&self,
|
||||
cell: &CellSpec,
|
||||
params: &[(String, Scalar)],
|
||||
window_ms: (i64, i64),
|
||||
) -> Result<RunReport, MemberFault>;
|
||||
}
|
||||
|
||||
/// Display-free member faults (the consumer phrases them — the RefFault
|
||||
/// pattern).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum MemberFault {
|
||||
NoData { instrument: String, window_ms: (i64, i64) },
|
||||
Bind(String),
|
||||
Run(String),
|
||||
}
|
||||
|
||||
/// Preflight + runtime refusals. Display-free and by-identifier: the
|
||||
/// consumer phrases them (the DocFault/RefFault pattern).
|
||||
#[derive(Debug)]
|
||||
pub enum ExecFault {
|
||||
/// v1 boundary: `std::monte_carlo` / `std::generalize` are not executable.
|
||||
UnsupportedStage { stage: usize, block: String },
|
||||
/// The pipeline is not `std::sweep (std::gate)* (std::walk_forward)?`.
|
||||
PipelineShape { detail: String },
|
||||
/// A sweep/walk_forward selection metric outside the registry's rankable
|
||||
/// roster.
|
||||
UnrankableMetric { stage: usize, metric: String },
|
||||
/// A gate predicate metric that is not a per-member scalar (e.g. an
|
||||
/// annotation name such as `deflated_score`).
|
||||
GateMetricNotPerMember { stage: usize, metric: String },
|
||||
/// `plateau:*` selection in walk_forward (a gated survivor subset has no
|
||||
/// grid lattice to smooth over).
|
||||
PlateauInWalkForward { stage: usize },
|
||||
/// sweep `deflate: true` with a non-argmax select rule.
|
||||
DeflatePlateauConflict { stage: usize },
|
||||
/// `WindowRoller` construction refusals at runtime.
|
||||
Window { stage: usize, detail: String },
|
||||
Member(MemberFault),
|
||||
Registry(RegistryError),
|
||||
}
|
||||
|
||||
/// The 14 per-member scalars a gate predicate may reference: the 3
|
||||
/// `RunMetrics` scalars plus the 11 `RMetrics` scalars. The three
|
||||
/// selection-annotation names (`deflated_score` / `overfit_probability` /
|
||||
/// `neighbourhood_score`) are deliberately NOT here — they describe a
|
||||
/// selection, not a member. Hand-copied roster (the third metric-roster site
|
||||
/// beside aura-research's 17-name vocabulary and aura-registry's rankable
|
||||
/// set, #190); drift fails safe: an unknown name is a preflight refusal,
|
||||
/// never a wrong number.
|
||||
pub const PER_MEMBER_METRICS: &[&str] = &[
|
||||
"total_pips", "max_drawdown", "bias_sign_flips",
|
||||
"expectancy_r", "n_trades", "win_rate", "avg_win_r", "avg_loss_r",
|
||||
"profit_factor", "max_r_drawdown", "n_open_at_end", "sqn_normalized",
|
||||
"sqn", "net_expectancy_r",
|
||||
];
|
||||
|
||||
/// The registry's rankable roster (`resolve_metric`'s name set) — the metrics
|
||||
/// a sweep/walk_forward stage may select on. Hand-copied (#190); drift fails
|
||||
/// safe (a name the registry would refuse is refused here first, before any
|
||||
/// member runs).
|
||||
pub const RANKABLE_METRICS: &[&str] = &[
|
||||
"total_pips", "max_drawdown", "bias_sign_flips",
|
||||
"sqn", "sqn_normalized", "expectancy_r", "net_expectancy_r",
|
||||
];
|
||||
|
||||
/// Deflation resample count — the shipped CLI `select_winner` constant.
|
||||
/// Duplicated against `aura-cli`'s private copy, tracked in #199.
|
||||
pub const DEFLATION_N_RESAMPLES: usize = 1000;
|
||||
/// Deflation moving-block length — the shipped CLI `select_winner` constant.
|
||||
/// Duplicated against `aura-cli`'s private copy, tracked in #199.
|
||||
pub const DEFLATION_BLOCK_LEN: usize = 5;
|
||||
|
||||
/// Resolve one of the 14 [`PER_MEMBER_METRICS`] against a member's report.
|
||||
/// An R-metric name against `metrics.r == None` reads `None` (conservative
|
||||
/// and deterministic: a gate predicate over `None` fails the member).
|
||||
/// Annotation names and unknown names read `None`.
|
||||
pub fn member_metric(report: &RunReport, name: &str) -> Option<f64> {
|
||||
let m = &report.metrics;
|
||||
match name {
|
||||
"total_pips" => Some(m.total_pips),
|
||||
"max_drawdown" => Some(m.max_drawdown),
|
||||
"bias_sign_flips" => Some(m.bias_sign_flips as f64),
|
||||
_ => {
|
||||
let r = m.r.as_ref()?;
|
||||
match name {
|
||||
"expectancy_r" => Some(r.expectancy_r),
|
||||
"n_trades" => Some(r.n_trades as f64),
|
||||
"win_rate" => Some(r.win_rate),
|
||||
"avg_win_r" => Some(r.avg_win_r),
|
||||
"avg_loss_r" => Some(r.avg_loss_r),
|
||||
"profit_factor" => Some(r.profit_factor),
|
||||
"max_r_drawdown" => Some(r.max_r_drawdown),
|
||||
"n_open_at_end" => Some(r.n_open_at_end as f64),
|
||||
"sqn_normalized" => Some(r.sqn_normalized),
|
||||
"sqn" => Some(r.sqn),
|
||||
"net_expectancy_r" => Some(r.net_expectancy_r),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `value <cmp> threshold` holds — the gate's comparator arm.
|
||||
// Consumed by `execute`'s gate stage; the allow drops once that lands.
|
||||
#[allow(dead_code)]
|
||||
fn predicate_holds(cmp: &Cmp, value: f64, threshold: f64) -> bool {
|
||||
match cmp {
|
||||
Cmp::Gt => value > threshold,
|
||||
Cmp::Ge => value >= threshold,
|
||||
Cmp::Lt => value < threshold,
|
||||
Cmp::Le => value <= threshold,
|
||||
}
|
||||
}
|
||||
|
||||
/// Statically refuse everything refusable before any member runs (the F7
|
||||
/// lesson applied forward): the v1 executable pipeline shape is exactly
|
||||
/// `std::sweep (std::gate)* (std::walk_forward)?`; every sweep/walk_forward
|
||||
/// selection metric is in [`RANKABLE_METRICS`]; every gate predicate metric
|
||||
/// is in [`PER_MEMBER_METRICS`]; walk_forward must not select `plateau:*`
|
||||
/// (a gated survivor subset has no grid lattice); sweep `deflate: true`
|
||||
/// composes only with `argmax`; walk_forward lengths must fit `i64` (the
|
||||
/// roller's Timestamp unit). The campaign parameter is the seam for
|
||||
/// campaign-level static checks (none in v1, hence unused).
|
||||
// Consumed by `execute`; the allow drops once that lands.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn preflight(process: &ProcessDoc, _campaign: &CampaignDoc) -> Result<(), ExecFault> {
|
||||
// v1 boundary first: an mc/generalize stage refuses wherever it sits,
|
||||
// before any shape complaint about the same stage.
|
||||
for (i, stage) in process.pipeline.iter().enumerate() {
|
||||
let block = match stage {
|
||||
StageBlock::MonteCarlo { .. } => "std::monte_carlo",
|
||||
StageBlock::Generalize { .. } => "std::generalize",
|
||||
_ => continue,
|
||||
};
|
||||
return Err(ExecFault::UnsupportedStage { stage: i, block: block.to_string() });
|
||||
}
|
||||
// shape: exactly `std::sweep (std::gate)* (std::walk_forward)?`.
|
||||
if !matches!(process.pipeline.first(), Some(StageBlock::Sweep { .. })) {
|
||||
return Err(ExecFault::PipelineShape {
|
||||
detail: "the first stage must be std::sweep".to_string(),
|
||||
});
|
||||
}
|
||||
let last = process.pipeline.len() - 1;
|
||||
for (i, stage) in process.pipeline.iter().enumerate().skip(1) {
|
||||
match stage {
|
||||
StageBlock::Sweep { .. } => {
|
||||
return Err(ExecFault::PipelineShape {
|
||||
detail: format!("stage {i}: only the first stage may be std::sweep"),
|
||||
});
|
||||
}
|
||||
StageBlock::WalkForward { .. } if i != last => {
|
||||
return Err(ExecFault::PipelineShape {
|
||||
detail: format!("stage {i}: std::walk_forward must be the final stage"),
|
||||
});
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
// per-stage slot rules (shape already established above).
|
||||
for (i, stage) in process.pipeline.iter().enumerate() {
|
||||
match stage {
|
||||
StageBlock::Sweep { metric, select, deflate } => {
|
||||
if !RANKABLE_METRICS.contains(&metric.as_str()) {
|
||||
return Err(ExecFault::UnrankableMetric { stage: i, metric: metric.clone() });
|
||||
}
|
||||
if *deflate && *select != SelectRule::Argmax {
|
||||
return Err(ExecFault::DeflatePlateauConflict { stage: i });
|
||||
}
|
||||
}
|
||||
StageBlock::Gate { all } => {
|
||||
for p in all {
|
||||
if !PER_MEMBER_METRICS.contains(&p.metric.as_str()) {
|
||||
return Err(ExecFault::GateMetricNotPerMember {
|
||||
stage: i,
|
||||
metric: p.metric.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
StageBlock::WalkForward {
|
||||
in_sample_ms,
|
||||
out_of_sample_ms,
|
||||
step_ms,
|
||||
mode: _,
|
||||
metric,
|
||||
select,
|
||||
} => {
|
||||
if !RANKABLE_METRICS.contains(&metric.as_str()) {
|
||||
return Err(ExecFault::UnrankableMetric { stage: i, metric: metric.clone() });
|
||||
}
|
||||
if matches!(select, SelectRule::PlateauMean | SelectRule::PlateauWorst) {
|
||||
return Err(ExecFault::PlateauInWalkForward { stage: i });
|
||||
}
|
||||
for (field, len) in [
|
||||
("in_sample_ms", *in_sample_ms),
|
||||
("out_of_sample_ms", *out_of_sample_ms),
|
||||
("step_ms", *step_ms),
|
||||
] {
|
||||
if i64::try_from(len).is_err() {
|
||||
return Err(ExecFault::PipelineShape {
|
||||
detail: format!("stage {i}: walk_forward {field} does not fit i64"),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
StageBlock::MonteCarlo { .. } | StageBlock::Generalize { .. } => {
|
||||
unreachable!("refused by the v1 scan above")
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aura_engine::{RMetrics, RunManifest, RunMetrics, Timestamp};
|
||||
|
||||
/// A report with every per-member scalar planted to a distinct value
|
||||
/// (1..=14 in `PER_MEMBER_METRICS` order), r block present.
|
||||
fn report_with_r() -> RunReport {
|
||||
RunReport {
|
||||
manifest: RunManifest {
|
||||
commit: "test".to_string(),
|
||||
params: vec![],
|
||||
window: (Timestamp(0), Timestamp(1)),
|
||||
seed: 0,
|
||||
broker: "sim".to_string(),
|
||||
selection: None,
|
||||
instrument: None,
|
||||
topology_hash: None,
|
||||
project: None,
|
||||
},
|
||||
metrics: RunMetrics {
|
||||
total_pips: 1.0,
|
||||
max_drawdown: 2.0,
|
||||
bias_sign_flips: 3,
|
||||
r: Some(RMetrics {
|
||||
expectancy_r: 4.0,
|
||||
n_trades: 5,
|
||||
win_rate: 6.0,
|
||||
avg_win_r: 7.0,
|
||||
avg_loss_r: 8.0,
|
||||
profit_factor: 9.0,
|
||||
max_r_drawdown: 10.0,
|
||||
n_open_at_end: 11,
|
||||
sqn_normalized: 12.0,
|
||||
sqn: 13.0,
|
||||
net_expectancy_r: 14.0,
|
||||
conviction_terciles_r: [0.0; 3],
|
||||
trade_rs: Vec::new(),
|
||||
}),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn member_metric_resolves_all_fourteen_names() {
|
||||
let rep = report_with_r();
|
||||
let expected: &[(&str, f64)] = &[
|
||||
("total_pips", 1.0),
|
||||
("max_drawdown", 2.0),
|
||||
("bias_sign_flips", 3.0),
|
||||
("expectancy_r", 4.0),
|
||||
("n_trades", 5.0),
|
||||
("win_rate", 6.0),
|
||||
("avg_win_r", 7.0),
|
||||
("avg_loss_r", 8.0),
|
||||
("profit_factor", 9.0),
|
||||
("max_r_drawdown", 10.0),
|
||||
("n_open_at_end", 11.0),
|
||||
("sqn_normalized", 12.0),
|
||||
("sqn", 13.0),
|
||||
("net_expectancy_r", 14.0),
|
||||
];
|
||||
assert_eq!(expected.len(), 14);
|
||||
assert_eq!(PER_MEMBER_METRICS.len(), 14);
|
||||
for (name, want) in expected {
|
||||
assert!(
|
||||
PER_MEMBER_METRICS.contains(name),
|
||||
"{name} missing from PER_MEMBER_METRICS"
|
||||
);
|
||||
assert_eq!(member_metric(&rep, name), Some(*want), "metric {name}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn member_metric_r_names_none_without_r_block() {
|
||||
let mut rep = report_with_r();
|
||||
rep.metrics.r = None;
|
||||
// the three run-level scalars still resolve...
|
||||
assert_eq!(member_metric(&rep, "total_pips"), Some(1.0));
|
||||
assert_eq!(member_metric(&rep, "max_drawdown"), Some(2.0));
|
||||
assert_eq!(member_metric(&rep, "bias_sign_flips"), Some(3.0));
|
||||
// ...and every R name reads None (a gate predicate over it fails).
|
||||
for name in [
|
||||
"expectancy_r", "n_trades", "win_rate", "avg_win_r", "avg_loss_r",
|
||||
"profit_factor", "max_r_drawdown", "n_open_at_end", "sqn_normalized",
|
||||
"sqn", "net_expectancy_r",
|
||||
] {
|
||||
assert_eq!(member_metric(&rep, name), None, "R metric {name} without r block");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn member_metric_refuses_annotation_and_unknown_names() {
|
||||
let rep = report_with_r();
|
||||
for name in [
|
||||
"deflated_score", "overfit_probability", "neighbourhood_score", "no_such_metric",
|
||||
] {
|
||||
assert_eq!(member_metric(&rep, name), None, "{name} must not resolve");
|
||||
assert!(
|
||||
!PER_MEMBER_METRICS.contains(&name),
|
||||
"{name} must not be in PER_MEMBER_METRICS"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn predicate_holds_covers_all_four_cmps() {
|
||||
assert!(predicate_holds(&Cmp::Gt, 1.0, 0.0));
|
||||
assert!(!predicate_holds(&Cmp::Gt, 0.0, 0.0));
|
||||
assert!(predicate_holds(&Cmp::Ge, 0.0, 0.0));
|
||||
assert!(!predicate_holds(&Cmp::Ge, -0.1, 0.0));
|
||||
assert!(predicate_holds(&Cmp::Lt, -1.0, 0.0));
|
||||
assert!(!predicate_holds(&Cmp::Lt, 0.0, 0.0));
|
||||
assert!(predicate_holds(&Cmp::Le, 0.0, 0.0));
|
||||
assert!(!predicate_holds(&Cmp::Le, 0.1, 0.0));
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// preflight
|
||||
// -----------------------------------------------------------------
|
||||
|
||||
use aura_core::ScalarKind;
|
||||
use aura_research::{
|
||||
DataSection, DocKind, DocRef, Predicate, Presentation, ProcessRef, StrategyEntry, WfMode,
|
||||
Window,
|
||||
};
|
||||
|
||||
fn process_of(pipeline: Vec<StageBlock>) -> ProcessDoc {
|
||||
ProcessDoc {
|
||||
format_version: 1,
|
||||
kind: DocKind::Process,
|
||||
name: "p".to_string(),
|
||||
description: None,
|
||||
pipeline,
|
||||
}
|
||||
}
|
||||
|
||||
/// A minimal intrinsically-valid campaign; preflight's campaign parameter
|
||||
/// carries no v1 rules, so one fixture serves every test.
|
||||
fn campaign() -> CampaignDoc {
|
||||
CampaignDoc {
|
||||
format_version: 1,
|
||||
kind: DocKind::Campaign,
|
||||
name: "c".to_string(),
|
||||
description: None,
|
||||
data: DataSection {
|
||||
instruments: vec!["EURUSD".to_string()],
|
||||
windows: vec![Window { from_ms: 0, to_ms: 10_000 }],
|
||||
},
|
||||
strategies: vec![StrategyEntry {
|
||||
r#ref: DocRef::ContentId("0".repeat(64)),
|
||||
axes: BTreeMap::from([(
|
||||
"len".to_string(),
|
||||
Axis {
|
||||
kind: ScalarKind::I64,
|
||||
values: vec![Scalar::i64(2), Scalar::i64(3)],
|
||||
},
|
||||
)]),
|
||||
}],
|
||||
process: ProcessRef { r#ref: DocRef::ContentId("1".repeat(64)) },
|
||||
seed: 7,
|
||||
presentation: Presentation { persist_taps: vec![], emit: vec![] },
|
||||
}
|
||||
}
|
||||
|
||||
fn sweep_stage(metric: &str, select: SelectRule, deflate: bool) -> StageBlock {
|
||||
StageBlock::Sweep { metric: metric.to_string(), select, deflate }
|
||||
}
|
||||
|
||||
fn gate_stage(metric: &str) -> StageBlock {
|
||||
StageBlock::Gate {
|
||||
all: vec![Predicate { metric: metric.to_string(), cmp: Cmp::Gt, value: 0.0 }],
|
||||
}
|
||||
}
|
||||
|
||||
fn wf_stage(metric: &str, select: SelectRule) -> StageBlock {
|
||||
StageBlock::WalkForward {
|
||||
in_sample_ms: 4000,
|
||||
out_of_sample_ms: 1000,
|
||||
step_ms: 1000,
|
||||
mode: WfMode::Rolling,
|
||||
metric: metric.to_string(),
|
||||
select,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preflight_accepts_the_v1_shape() {
|
||||
let c = campaign();
|
||||
// the full v1 shape: sweep (gate)* (walk_forward)?
|
||||
let full = process_of(vec![
|
||||
sweep_stage("sqn_normalized", SelectRule::Argmax, true),
|
||||
gate_stage("net_expectancy_r"),
|
||||
wf_stage("sqn_normalized", SelectRule::Argmax),
|
||||
]);
|
||||
assert!(preflight(&full, &c).is_ok());
|
||||
// degenerate accepted shapes: bare sweep (plateau select without
|
||||
// deflate is legal on a sweep); sweep + gates without walk_forward.
|
||||
let bare = process_of(vec![sweep_stage("total_pips", SelectRule::PlateauMean, false)]);
|
||||
assert!(preflight(&bare, &c).is_ok());
|
||||
let gated = process_of(vec![
|
||||
sweep_stage("expectancy_r", SelectRule::Argmax, false),
|
||||
gate_stage("n_trades"),
|
||||
gate_stage("win_rate"),
|
||||
]);
|
||||
assert!(preflight(&gated, &c).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preflight_refuses_mc_and_generalize_stages() {
|
||||
let c = campaign();
|
||||
let mc = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
StageBlock::MonteCarlo { resamples: 100, block_len: 5 },
|
||||
]);
|
||||
assert!(matches!(
|
||||
preflight(&mc, &c),
|
||||
Err(ExecFault::UnsupportedStage { stage: 1, block }) if block == "std::monte_carlo"
|
||||
));
|
||||
let g = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
StageBlock::Generalize { metric: "expectancy_r".to_string() },
|
||||
]);
|
||||
assert!(matches!(
|
||||
preflight(&g, &c),
|
||||
Err(ExecFault::UnsupportedStage { stage: 1, block }) if block == "std::generalize"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preflight_refuses_non_sweep_first_and_double_sweep() {
|
||||
let c = campaign();
|
||||
let gate_first = process_of(vec![gate_stage("expectancy_r")]);
|
||||
assert!(matches!(preflight(&gate_first, &c), Err(ExecFault::PipelineShape { .. })));
|
||||
let empty = process_of(vec![]);
|
||||
assert!(matches!(preflight(&empty, &c), Err(ExecFault::PipelineShape { .. })));
|
||||
let double = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
]);
|
||||
assert!(matches!(preflight(&double, &c), Err(ExecFault::PipelineShape { .. })));
|
||||
// walk_forward anywhere but last is a shape refusal too.
|
||||
let wf_mid = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
wf_stage("sqn", SelectRule::Argmax),
|
||||
gate_stage("expectancy_r"),
|
||||
]);
|
||||
assert!(matches!(preflight(&wf_mid, &c), Err(ExecFault::PipelineShape { .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preflight_refuses_unrankable_select_metric() {
|
||||
let c = campaign();
|
||||
// win_rate is a per-member scalar but NOT in the registry's rankable roster.
|
||||
let sweep_bad = process_of(vec![sweep_stage("win_rate", SelectRule::Argmax, false)]);
|
||||
assert!(matches!(
|
||||
preflight(&sweep_bad, &c),
|
||||
Err(ExecFault::UnrankableMetric { stage: 0, metric }) if metric == "win_rate"
|
||||
));
|
||||
let wf_bad = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
wf_stage("profit_factor", SelectRule::Argmax),
|
||||
]);
|
||||
assert!(matches!(
|
||||
preflight(&wf_bad, &c),
|
||||
Err(ExecFault::UnrankableMetric { stage: 1, metric }) if metric == "profit_factor"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preflight_refuses_annotation_gate_metric() {
|
||||
let c = campaign();
|
||||
let p = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
gate_stage("deflated_score"),
|
||||
]);
|
||||
assert!(matches!(
|
||||
preflight(&p, &c),
|
||||
Err(ExecFault::GateMetricNotPerMember { stage: 1, metric }) if metric == "deflated_score"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preflight_refuses_plateau_in_walk_forward() {
|
||||
let c = campaign();
|
||||
for select in [SelectRule::PlateauMean, SelectRule::PlateauWorst] {
|
||||
let p = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
wf_stage("sqn", select),
|
||||
]);
|
||||
assert!(matches!(
|
||||
preflight(&p, &c),
|
||||
Err(ExecFault::PlateauInWalkForward { stage: 1 })
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preflight_refuses_wf_length_exceeding_i64() {
|
||||
let c = campaign();
|
||||
let p = process_of(vec![
|
||||
sweep_stage("sqn", SelectRule::Argmax, false),
|
||||
StageBlock::WalkForward {
|
||||
in_sample_ms: u64::MAX,
|
||||
out_of_sample_ms: 1000,
|
||||
step_ms: 1000,
|
||||
mode: WfMode::Rolling,
|
||||
metric: "sqn".to_string(),
|
||||
select: SelectRule::Argmax,
|
||||
},
|
||||
]);
|
||||
assert!(matches!(
|
||||
preflight(&p, &c),
|
||||
Err(ExecFault::PipelineShape { detail })
|
||||
if detail.contains("in_sample_ms does not fit i64")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preflight_refuses_deflate_with_plateau() {
|
||||
let c = campaign();
|
||||
for select in [SelectRule::PlateauMean, SelectRule::PlateauWorst] {
|
||||
let p = process_of(vec![sweep_stage("sqn", select, true)]);
|
||||
assert!(matches!(
|
||||
preflight(&p, &c),
|
||||
Err(ExecFault::DeflatePlateauConflict { stage: 0 })
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
//! End-to-end coverage for aura-campaign's public seam (cycle 0107 tasks
|
||||
//! 4-5): `MemberRunner` is documented as the ONE binding point every
|
||||
//! consumer implements ("every consumer ... binds its own runner while the
|
||||
//! execution semantics live here once"), so these tests drive it as an
|
||||
//! external consumer would — through `&dyn MemberRunner`, never an internal
|
||||
//! function — and check the seam's plumbing plus the cross-roster invariant
|
||||
//! `preflight` silently depends on. The in-crate unit tests in `lib.rs` pin
|
||||
//! `member_metric`'s full 14-name mapping and `preflight`'s (crate-private)
|
||||
//! shape rules against hand-built fixtures; these tests pin what a real,
|
||||
//! external, dynamically-dispatched consumer sees.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aura_campaign::{member_metric, CellSpec, MemberFault, MemberRunner, PER_MEMBER_METRICS, RANKABLE_METRICS};
|
||||
use aura_engine::{summarize, RMetrics, RunManifest, RunMetrics, RunReport, Scalar, Timestamp};
|
||||
|
||||
fn cell() -> CellSpec {
|
||||
CellSpec {
|
||||
strategy_ordinal: 0,
|
||||
strategy_id: "0".repeat(64),
|
||||
blueprint_json: "{}".to_string(),
|
||||
axes: BTreeMap::new(),
|
||||
instrument: "EURUSD".to_string(),
|
||||
window_ms: (0, 20),
|
||||
}
|
||||
}
|
||||
|
||||
/// A minimal external `MemberRunner`: ignores its params and produces
|
||||
/// metrics via the real `aura_engine::summarize` reduction (not a hand-typed
|
||||
/// `RunMetrics` literal), echoing the `window_ms` it was called with into
|
||||
/// the manifest — exactly what a real consumer's harness-binding code does.
|
||||
struct FixedRunner;
|
||||
|
||||
impl MemberRunner for FixedRunner {
|
||||
fn run_member(
|
||||
&self,
|
||||
_cell: &CellSpec,
|
||||
_params: &[(String, Scalar)],
|
||||
window_ms: (i64, i64),
|
||||
) -> Result<RunReport, MemberFault> {
|
||||
let equity = vec![(Timestamp(0), 0.0), (Timestamp(1), 3.0), (Timestamp(2), 1.0)];
|
||||
let metrics = summarize(&equity, &[]);
|
||||
Ok(RunReport {
|
||||
manifest: RunManifest {
|
||||
commit: "e2e".to_string(),
|
||||
params: vec![],
|
||||
window: (Timestamp(window_ms.0), Timestamp(window_ms.1)),
|
||||
seed: 0,
|
||||
broker: "test".to_string(),
|
||||
selection: None,
|
||||
instrument: None,
|
||||
topology_hash: None,
|
||||
project: None,
|
||||
},
|
||||
metrics,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Property: `MemberRunner` is object-safe and dispatches through
|
||||
/// `&dyn MemberRunner` (the shape a heterogeneous, per-consumer roster of
|
||||
/// runners requires); the exact sub-window given at the call site — not
|
||||
/// `cell.window_ms`, a walk-forward stage passes a narrower sub-range —
|
||||
/// reaches the runner unmodified; and the real `aura_engine::summarize()`
|
||||
/// output the runner produced is what `member_metric` reads back through the
|
||||
/// crate's public per-name resolver.
|
||||
#[test]
|
||||
fn member_runner_seam_dispatches_through_dyn_trait_object_and_carries_the_window() {
|
||||
let runner = FixedRunner;
|
||||
let dyn_runner: &dyn MemberRunner = &runner;
|
||||
let report = dyn_runner
|
||||
.run_member(&cell(), &[("len".to_string(), Scalar::i64(3))], (10, 20))
|
||||
.expect("member run succeeds");
|
||||
|
||||
// the call-site sub-window (10, 20), not cell().window_ms == (0, 20),
|
||||
// reached the runner unmodified
|
||||
assert_eq!(report.manifest.window, (Timestamp(10), Timestamp(20)));
|
||||
|
||||
// the real summarize() reduction over [0.0, 3.0, 1.0] is what
|
||||
// member_metric reads back: last == 1.0, peak-to-trough == 2.0
|
||||
assert_eq!(member_metric(&report, "total_pips"), Some(1.0));
|
||||
assert_eq!(member_metric(&report, "max_drawdown"), Some(2.0));
|
||||
}
|
||||
|
||||
/// Property: every metric name in `RANKABLE_METRICS` (the roster `preflight`
|
||||
/// permits a `std::sweep`/`std::walk_forward` stage to SELECT on) is also a
|
||||
/// member of `PER_MEMBER_METRICS` and resolves to `Some` via `member_metric`.
|
||||
/// The two rosters are independently hand-copied (documented drift risk,
|
||||
/// #190); were they to diverge, `preflight` would accept a selection metric
|
||||
/// that `member_metric` silently reads back as `None` for every member — a
|
||||
/// sweep/walk_forward stage that always empties, not a compile error or an
|
||||
/// early refusal.
|
||||
#[test]
|
||||
fn rankable_metrics_are_all_per_member_resolvable() {
|
||||
let report = RunReport {
|
||||
manifest: RunManifest {
|
||||
commit: "e2e".to_string(),
|
||||
params: vec![],
|
||||
window: (Timestamp(0), Timestamp(1)),
|
||||
seed: 0,
|
||||
broker: "test".to_string(),
|
||||
selection: None,
|
||||
instrument: None,
|
||||
topology_hash: None,
|
||||
project: None,
|
||||
},
|
||||
metrics: RunMetrics {
|
||||
total_pips: 1.0,
|
||||
max_drawdown: 2.0,
|
||||
bias_sign_flips: 3,
|
||||
r: Some(RMetrics {
|
||||
expectancy_r: 4.0,
|
||||
n_trades: 5,
|
||||
win_rate: 6.0,
|
||||
avg_win_r: 7.0,
|
||||
avg_loss_r: 8.0,
|
||||
profit_factor: 9.0,
|
||||
max_r_drawdown: 10.0,
|
||||
n_open_at_end: 11,
|
||||
sqn_normalized: 12.0,
|
||||
sqn: 13.0,
|
||||
net_expectancy_r: 14.0,
|
||||
conviction_terciles_r: [0.0; 3],
|
||||
trade_rs: Vec::new(),
|
||||
}),
|
||||
},
|
||||
};
|
||||
for &metric in RANKABLE_METRICS {
|
||||
assert!(
|
||||
PER_MEMBER_METRICS.contains(&metric),
|
||||
"{metric} is rankable but missing from PER_MEMBER_METRICS"
|
||||
);
|
||||
assert!(
|
||||
member_metric(&report, metric).is_some(),
|
||||
"{metric} is rankable but member_metric returned None"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user