refactor: cut the engine's backtest-metrics edge via RunReport<M>
C28 phase 2 (Stratification); realizes item 1 of the deferred #147. The engine's production surface no longer names a backtest-metric type: - RunReport becomes generic over its metric payload M; sweep/mc/walkforward/ blueprint thread the parameter (SweepPoint<M>, SweepFamily<M>, WindowRun<M>, WalkForwardResult<M>). RunManifest stays concrete and engine-owned (its selection: Option<FamilySelection> embeds the foundation-grade analysis type). - summarize and the MC assembly (McDraw/McFamily/McAggregate/RBootstrap/ r_bootstrap/monte_carlo) move to aura-backtest - McAggregate::from_draws reads RunMetrics fields by name, so generifying it is the phase-6 metric-vocabulary abstraction (#147 item 2), still deferred; wholesale relocation is the honest cut. The concrete instantiation lives in aura-backtest as `type RunReport = aura_engine::RunReport<RunMetrics>` + sibling aliases. - the statistics kernel (MetricStats/quantile/resample_block/SplitMix64) moves to the aura-analysis foundation; the engine re-imports it (inner->foundation, legal) and re-exports it so existing consumers stay source-compatible. Dependency inversion in one commit: aura-engine drops aura-backtest from [dependencies] (back to dev-deps for its SimBroker/RunMetrics test fixtures); aura-backtest gains aura-engine. Cycle-free for lib targets - the cycle closes only through the engine's dev-dep edge, the pattern aura-vocabulary already uses. aura-backtest reaches the kernel transitively through the engine re-export, so no aura-backtest -> aura-analysis edge exists (the C28 ladder permits backtest -> {core, engine} only). run_indexed / SplitMix64::next_f64 widened pub(crate) -> pub for cross-crate use. Consumers (registry/campaign/cli/composites/ingest/bench) rewired by import path only, no call-site logic changed. The c28_layering structural test extends to the full ladder: aura-analysis (no aura-* deps), aura-engine ⊆ {core, analysis}, aura-backtest ⊆ {core, engine}. Behaviour-preserving: 1448/0 tests, clippy -D warnings clean, serde shapes byte-identical (C18 - RunReport<M> keeps field order manifest,metrics; the CLI pre-serialized-splice contract unchanged), moved code traceable via git rename detection. Cycle-introduced broken intra-doc links fixed. closes #292
This commit is contained in:
@@ -7,6 +7,14 @@ publish.workspace = true
|
||||
|
||||
[dependencies]
|
||||
aura-core = { path = "../aura-core" }
|
||||
# aura-engine is the metric-generic engine this crate instantiates with
|
||||
# `M = RunMetrics` (`RunReport`/`SweepFamily`/etc. aliases), whose
|
||||
# `run_indexed` the moved MC assembly drives, and whose re-exported
|
||||
# `MetricStats`/`resample_block`/`SplitMix64` (foundation-grade, from
|
||||
# aura-analysis) the moved `mc.rs` reduces through — no direct aura-analysis
|
||||
# edge, keeping the C28 ladder one-hop (#292, C28 phase 2). Outer-rung
|
||||
# dependency — the engine itself only dev-depends back on this crate.
|
||||
aura-engine = { path = "../aura-engine" }
|
||||
# serde gives the backtest metric types (`RunMetrics`/`RMetrics`/`PositionEvent`)
|
||||
# their typed (de)serialization path for the run registry (C18; moved with the
|
||||
# types from aura-analysis, #291). Deterministic output (C1-safe).
|
||||
@@ -17,3 +25,13 @@ serde = { workspace = true }
|
||||
# for production lives in aura-engine's RunReport::to_json). Test-only, so it
|
||||
# never enters the frozen artifact's library link (C16).
|
||||
serde_json = { workspace = true }
|
||||
# aura-std / aura-strategy back the moved `mc.rs`/`metrics.rs` test fixtures
|
||||
# (Sma/Sub/Recorder, Bias) — the crate-private `aura-engine::test_fixtures`
|
||||
# harness is unreachable from here, so the fixture is reconstructed through
|
||||
# the public node vocabulary instead (test-only, mirrors the aura-engine
|
||||
# integration tests' own pattern).
|
||||
aura-std = { path = "../aura-std" }
|
||||
aura-strategy = { path = "../aura-strategy" }
|
||||
# rayon backs the thread-count-explicit `monte_carlo_with_threads` test
|
||||
# wrapper (moved verbatim from aura-engine's mc.rs, #291) — test-only.
|
||||
rayon = "1"
|
||||
|
||||
@@ -1,15 +1,22 @@
|
||||
//! `aura-backtest` — the backtest layer (C28): simulated execution without
|
||||
//! money (the `SimBroker` pip yardstick, position management), measured in R,
|
||||
//! plus the backtest metric reductions over the recorded streams
|
||||
//! (`metrics`, moved from `aura-analysis` by #291). Depends only on
|
||||
//! `aura-core` (+ serde for the metric types' C18 wire shapes).
|
||||
//! (`metrics`, moved from `aura-analysis` by #291) and the Monte-Carlo
|
||||
//! assembly (`mc`, moved from `aura-engine` by #291/#292, C28 phase 2). The
|
||||
//! outer rung of the metric-generic engine (C28): `RunReport` here is the
|
||||
//! concrete `M = RunMetrics` instantiation of `aura_engine::RunReport<M>`.
|
||||
|
||||
mod mc;
|
||||
mod metrics;
|
||||
mod position_management;
|
||||
mod sim_broker;
|
||||
|
||||
// `assemble_mc` stays module-private (it was never `pub` in the pre-#291 engine
|
||||
// lib.rs re-export either — only its two callers, `monte_carlo` and the
|
||||
// `#[cfg(test)]` thread-count wrapper, are public/test surface).
|
||||
pub use mc::{monte_carlo, r_bootstrap, McAggregate, McDraw, McFamily, RBootstrap};
|
||||
pub use metrics::{
|
||||
derive_position_events, r_metrics_from_rs, summarize_r, PositionAction,
|
||||
derive_position_events, r_metrics_from_rs, summarize, summarize_r, PositionAction,
|
||||
PositionEvent, RMetrics, RunMetrics,
|
||||
};
|
||||
pub use position_management::{
|
||||
@@ -17,3 +24,12 @@ pub use position_management::{
|
||||
RECORD_KINDS as PM_RECORD_KINDS, WIDTH as PM_WIDTH,
|
||||
};
|
||||
pub use sim_broker::SimBroker;
|
||||
|
||||
/// The trading instantiation of the engine's metric-generic run record (C28
|
||||
/// phase 2, #292): source-compatible concrete names for every consumer that
|
||||
/// deals in backtest metrics.
|
||||
pub type RunReport = aura_engine::RunReport<RunMetrics>;
|
||||
pub type SweepPoint = aura_engine::SweepPoint<RunMetrics>;
|
||||
pub type SweepFamily = aura_engine::SweepFamily<RunMetrics>;
|
||||
pub type WindowRun = aura_engine::WindowRun<RunMetrics>;
|
||||
pub type WalkForwardResult = aura_engine::WalkForwardResult<RunMetrics>;
|
||||
|
||||
@@ -0,0 +1,483 @@
|
||||
//! Monte-Carlo orchestration family (C12 axis 4): Monte-Carlo as a sweep over
|
||||
//! seeds. `monte_carlo(base_point, seeds, run_one)` runs a fixed base point over
|
||||
//! a seed set — each seed a disjoint C1 realization — and collects an
|
||||
//! [`McFamily`]: the per-seed [`McDraw`]s in seed-input order plus a stored
|
||||
//! [`McAggregate`] (mean + quantiles of all three run metrics across the
|
||||
//! realizations). It reuses the disjoint-parallel [`run_indexed`](aura_engine::run_indexed)
|
||||
//! core `sweep` drives — the varying dimension is the *seed*, not a tuning param.
|
||||
//! Eager-agnostic (C12/#71): the API takes seeds + a per-draw closure, never a
|
||||
//! materialized stream `Vec`; the seed -> `Source` construction is a closure-body
|
||||
//! concern. Moved verbatim from `aura-engine::mc` (#291, C28 phase 2): the
|
||||
//! engine stays metric-agnostic, so the concrete `RunMetrics` instantiation of
|
||||
//! `RunReport` lives here, the outer C28 rung.
|
||||
|
||||
// MetricStats/resample_block/SplitMix64 are foundation-grade (aura-analysis),
|
||||
// but reached here through aura-engine's re-export rather than a direct
|
||||
// aura-analysis dependency — the C28 ladder (c28_layering) permits
|
||||
// aura-backtest -> {aura-core, aura-engine} only, not a same-rung-skipping
|
||||
// direct edge to the foundation crate.
|
||||
use aura_core::Scalar;
|
||||
use aura_engine::{run_indexed, resample_block, MetricStats, SplitMix64};
|
||||
use crate::{RunMetrics, RunReport};
|
||||
|
||||
/// One Monte-Carlo realization: the seed that drove it and the full `RunReport`.
|
||||
/// Self-describing, analog to [`SweepPoint`](aura_engine::SweepPoint).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct McDraw {
|
||||
pub seed: u64,
|
||||
pub report: RunReport,
|
||||
}
|
||||
|
||||
/// The result family of a Monte-Carlo run — one [`McDraw`] per seed, in
|
||||
/// seed-**input** order (independent of thread completion), plus the stored
|
||||
/// [`McAggregate`]. Analog to [`SweepFamily`](aura_engine::SweepFamily).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct McFamily {
|
||||
pub draws: Vec<McDraw>,
|
||||
pub aggregate: McAggregate,
|
||||
}
|
||||
|
||||
/// Distribution summary of all three run metrics across the realizations: covers
|
||||
/// every metric (not a single "chosen" one). A pure post-run reduction over the
|
||||
/// draws — stored for the common robustness case; custom statistics read the raw
|
||||
/// draws directly.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct McAggregate {
|
||||
pub total_pips: MetricStats,
|
||||
pub max_drawdown: MetricStats,
|
||||
pub bias_sign_flips: MetricStats,
|
||||
}
|
||||
|
||||
impl McAggregate {
|
||||
/// Pure reduction over the realizations — recomputable from `draws` alone (it
|
||||
/// is exactly what [`monte_carlo`] stored). `draws` must be non-empty (a
|
||||
/// Monte-Carlo over zero realizations has no defined mean/quantile); on a
|
||||
/// non-empty slice every field is finite.
|
||||
pub fn from_draws(draws: &[McDraw]) -> McAggregate {
|
||||
let pick = |f: fn(&RunMetrics) -> f64| -> MetricStats {
|
||||
let xs: Vec<f64> = draws.iter().map(|d| f(&d.report.metrics)).collect();
|
||||
MetricStats::from_values(&xs)
|
||||
};
|
||||
McAggregate {
|
||||
total_pips: pick(|m| m.total_pips),
|
||||
max_drawdown: pick(|m| m.max_drawdown),
|
||||
bias_sign_flips: pick(|m| m.bias_sign_flips as f64),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The single assembly path behind both [`monte_carlo`] and the thread-count-
|
||||
/// explicit test wrapper: run `run_one` over every seed via `run_indexed` (on
|
||||
/// whichever pool is ambient when called), build the per-seed draws in
|
||||
/// seed-input order, and reduce them into the aggregate. One source of truth so
|
||||
/// the two callers can never drift on assembly logic while their pool selection
|
||||
/// differs.
|
||||
fn assemble_mc<F>(base_point: &[Scalar], seeds: &[u64], run_one: &F) -> McFamily
|
||||
where
|
||||
F: Fn(u64, &[Scalar]) -> RunReport + Sync,
|
||||
{
|
||||
let reports = run_indexed(seeds.len(), |i| run_one(seeds[i], base_point));
|
||||
let draws: Vec<McDraw> = seeds
|
||||
.iter()
|
||||
.zip(reports)
|
||||
.map(|(&seed, report)| McDraw { seed, report })
|
||||
.collect();
|
||||
let aggregate = McAggregate::from_draws(&draws);
|
||||
McFamily { draws, aggregate }
|
||||
}
|
||||
|
||||
/// Run `run_one(seed, base_point)` over every seed, disjointly in parallel (C1),
|
||||
/// and collect the family in seed-input order plus the aggregate. The varying
|
||||
/// dimension is the *seed* (C12 axis 4: MC = sweep over seeds), not a tuning
|
||||
/// param; `base_point` is constant across draws. Eager-agnostic: the seed ->
|
||||
/// `Source` construction lives inside `run_one`, never a materialized stream
|
||||
/// `Vec` in this API (#71).
|
||||
///
|
||||
/// Precondition: `seeds` is non-empty (a Monte-Carlo over zero realizations has
|
||||
/// no defined aggregate). A `debug_assert!` guards it; the `-> McFamily`
|
||||
/// signature is preserved (no `Result`), matching [`sweep`](aura_engine::sweep).
|
||||
pub fn monte_carlo<F>(base_point: &[Scalar], seeds: &[u64], run_one: F) -> McFamily
|
||||
where
|
||||
F: Fn(u64, &[Scalar]) -> RunReport + Sync,
|
||||
{
|
||||
debug_assert!(!seeds.is_empty(), "monte_carlo requires a non-empty seed set");
|
||||
assemble_mc(base_point, seeds, &run_one)
|
||||
}
|
||||
|
||||
/// The thread-count-explicit wrapper of [`monte_carlo`]. Module-private: the
|
||||
/// public `monte_carlo` runs on the ambient pool; the tests drive this at 1 and
|
||||
/// at N to pin determinism (C1). `assemble_mc` (run_indexed + draw assembly +
|
||||
/// aggregate) runs inside a local rayon pool of `nthreads` workers, sharing the
|
||||
/// exact assembly path `monte_carlo` uses — no second copy to drift. Bounds
|
||||
/// match `monte_carlo` (`F: Sync`): the install closure only borrows `run_one`,
|
||||
/// never moves it.
|
||||
#[cfg(test)]
|
||||
fn monte_carlo_with_threads<F>(
|
||||
base_point: &[Scalar],
|
||||
seeds: &[u64],
|
||||
nthreads: usize,
|
||||
run_one: F,
|
||||
) -> McFamily
|
||||
where
|
||||
F: Fn(u64, &[Scalar]) -> RunReport + Sync,
|
||||
{
|
||||
debug_assert!(!seeds.is_empty(), "monte_carlo requires a non-empty seed set");
|
||||
let pool = rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(nthreads)
|
||||
.build()
|
||||
.expect("rayon thread pool");
|
||||
pool.install(|| assemble_mc(base_point, seeds, &run_one))
|
||||
}
|
||||
|
||||
/// Distribution of `E[R]` under a moving-block bootstrap of an OOS per-trade R series
|
||||
/// (#139). `block_len == 1` is the i.i.d. trade-shuffle; `block_len > 1` resamples
|
||||
/// contiguous runs, preserving the serial correlation of sequential trades a pure
|
||||
/// shuffle would erase. Deterministic (C1) given `seed`.
|
||||
#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct RBootstrap {
|
||||
pub e_r: MetricStats, // mean + p5/p25/p50/p75/p95 of the resampled E[R]
|
||||
pub prob_le_zero: f64, // fraction of resamples whose mean R <= 0
|
||||
pub n_trades: usize,
|
||||
pub block_len: usize,
|
||||
pub n_resamples: usize,
|
||||
}
|
||||
|
||||
/// Moving-block bootstrap of `rs` (non-circular; the final block of each resample is
|
||||
/// truncated so the resample has exactly `n` values). `block_len` is clamped to
|
||||
/// `[1, n]`. Empty `rs` or zero resamples -> an all-zero `RBootstrap`. Pure given
|
||||
/// `seed` (drives the existing `SplitMix64`).
|
||||
pub fn r_bootstrap(rs: &[f64], n_resamples: usize, block_len: usize, seed: u64) -> RBootstrap {
|
||||
let n = rs.len();
|
||||
if n == 0 || n_resamples == 0 {
|
||||
return RBootstrap {
|
||||
e_r: MetricStats { mean: 0.0, p5: 0.0, p25: 0.0, p50: 0.0, p75: 0.0, p95: 0.0 },
|
||||
prob_le_zero: 0.0,
|
||||
n_trades: n,
|
||||
block_len: block_len.clamp(1, n.max(1)),
|
||||
n_resamples,
|
||||
};
|
||||
}
|
||||
let block_len = block_len.clamp(1, n);
|
||||
let mut rng = SplitMix64::new(seed);
|
||||
let mut means: Vec<f64> = Vec::with_capacity(n_resamples);
|
||||
for _ in 0..n_resamples {
|
||||
let sample = resample_block(rs, block_len, &mut rng);
|
||||
means.push(sample.iter().sum::<f64>() / n as f64);
|
||||
}
|
||||
let e_r = MetricStats::from_values(&means);
|
||||
let prob_le_zero = means.iter().filter(|&&m| m <= 0.0).count() as f64 / n_resamples as f64;
|
||||
RBootstrap { e_r, prob_le_zero, n_trades: n, block_len, n_resamples }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aura_engine::{
|
||||
f64_field, BlueprintNode, Composite, Edge, OutField, Role, RunManifest, SyntheticSpec,
|
||||
Target, Timestamp,
|
||||
};
|
||||
use aura_core::{Firing, Scalar, ScalarKind};
|
||||
use aura_std::{Recorder, Sma, Sub};
|
||||
use aura_strategy::Bias;
|
||||
use crate::{summarize, SimBroker};
|
||||
use std::sync::mpsc;
|
||||
|
||||
// The SMA-cross signal-quality harness fixture, reconstructed here through
|
||||
// the PUBLIC aura-engine/aura-std/aura-strategy/aura-backtest surface —
|
||||
// `aura-engine::test_fixtures` is `#[cfg(test)]`-private to the engine crate
|
||||
// itself and unreachable from here (the same reason `random_sweep_e2e.rs` /
|
||||
// `list_sweep_e2e.rs` carry their own copy). Bodies verbatim from
|
||||
// `aura-engine/src/test_fixtures.rs`'s `sma_cross`/`composite_sma_cross_harness`.
|
||||
|
||||
/// The SMA-cross signal as a reusable value-empty composite: one input role
|
||||
/// (price), one output (fast-minus-slow spread). Lengths injected at compile.
|
||||
fn sma_cross() -> Composite {
|
||||
Composite::new(
|
||||
"sma_cross",
|
||||
vec![
|
||||
Sma::builder().named("fast").into(),
|
||||
Sma::builder().named("slow").into(),
|
||||
Sub::builder().into(),
|
||||
],
|
||||
vec![
|
||||
Edge { from: 0, to: 2, slot: 0, from_field: 0 },
|
||||
Edge { from: 1, to: 2, slot: 1, from_field: 0 },
|
||||
],
|
||||
vec![Role {
|
||||
name: "price".into(),
|
||||
targets: vec![Target { node: 0, slot: 0 }, Target { node: 1, slot: 0 }],
|
||||
source: None,
|
||||
}],
|
||||
vec![OutField { node: 2, field: 0, name: "out".into() }],
|
||||
)
|
||||
}
|
||||
|
||||
/// The signal-quality harness as a value-empty composite blueprint with two
|
||||
/// recording sinks (equity, exposure).
|
||||
#[allow(clippy::type_complexity)]
|
||||
fn composite_sma_cross_harness() -> (
|
||||
Composite,
|
||||
mpsc::Receiver<(Timestamp, Vec<Scalar>)>,
|
||||
mpsc::Receiver<(Timestamp, Vec<Scalar>)>,
|
||||
) {
|
||||
let (tx_eq, rx_eq) = mpsc::channel();
|
||||
let (tx_ex, rx_ex) = mpsc::channel();
|
||||
let bp = Composite::new(
|
||||
"root",
|
||||
vec![
|
||||
BlueprintNode::Composite(sma_cross()),
|
||||
Bias::builder().named("bias").into(),
|
||||
SimBroker::builder(0.0001).into(),
|
||||
Recorder::builder(vec![ScalarKind::F64], Firing::Any, tx_eq).into(),
|
||||
Recorder::builder(vec![ScalarKind::F64], Firing::Any, tx_ex).into(),
|
||||
],
|
||||
vec![
|
||||
Edge { from: 0, to: 1, slot: 0, from_field: 0 }, // composite out -> Bias
|
||||
Edge { from: 1, to: 2, slot: 0, from_field: 0 }, // exposure -> broker slot 0
|
||||
Edge { from: 2, to: 3, slot: 0, from_field: 0 }, // equity -> sink
|
||||
Edge { from: 1, to: 4, slot: 0, from_field: 0 }, // exposure -> sink
|
||||
],
|
||||
vec![Role {
|
||||
name: "src".into(),
|
||||
targets: vec![
|
||||
Target { node: 0, slot: 0 }, // price -> sma_cross role 0
|
||||
Target { node: 2, slot: 1 }, // price -> SimBroker price slot
|
||||
],
|
||||
source: Some(ScalarKind::F64),
|
||||
}],
|
||||
vec![], // output: the root ends in sinks
|
||||
);
|
||||
(bp, rx_eq, rx_ex)
|
||||
}
|
||||
|
||||
/// Build + bootstrap + run + drain + summarize one (seed, point) into a
|
||||
/// `RunReport`. A free `fn` (Copy + Sync) so it serves as the `monte_carlo`
|
||||
/// closure AND a direct reference for the "draw == independent run"
|
||||
/// comparison. The stream is generated from `seed` (seed-as-input, #66), and
|
||||
/// `seed` is recorded into the manifest.
|
||||
fn run_draw(seed: u64, point: &[Scalar]) -> RunReport {
|
||||
let (bp, rx_eq, rx_ex) = composite_sma_cross_harness();
|
||||
let mut h = bp
|
||||
.bootstrap_with_params(point.to_vec())
|
||||
.expect("base point is kind-checked against param_space");
|
||||
let spec = SyntheticSpec { start: 1.0, len: 64, step: 1 };
|
||||
let window = (Timestamp(1), Timestamp((spec.len as i64 - 1) * spec.step + 1));
|
||||
h.run(vec![Box::new(spec.source(seed))]);
|
||||
let equity = f64_field(&rx_eq.try_iter().collect::<Vec<_>>(), 0);
|
||||
let exposure = f64_field(&rx_ex.try_iter().collect::<Vec<_>>(), 0);
|
||||
RunReport {
|
||||
manifest: RunManifest {
|
||||
commit: "test".to_string(),
|
||||
params: Vec::new(),
|
||||
defaults: Vec::new(),
|
||||
window,
|
||||
seed,
|
||||
broker: "test".to_string(),
|
||||
selection: None,
|
||||
instrument: None,
|
||||
topology_hash: None,
|
||||
project: None,
|
||||
},
|
||||
metrics: summarize(&equity, &exposure),
|
||||
}
|
||||
}
|
||||
|
||||
fn base_point() -> Vec<Scalar> {
|
||||
// matches the composite_sma_cross param_space order: fast, slow, scale
|
||||
vec![Scalar::i64(2), Scalar::i64(4), Scalar::f64(0.5)]
|
||||
}
|
||||
|
||||
fn draws_with_metrics(values: &[f64]) -> Vec<McDraw> {
|
||||
values
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, &v)| McDraw {
|
||||
seed: i as u64,
|
||||
report: RunReport {
|
||||
manifest: RunManifest {
|
||||
commit: "t".to_string(),
|
||||
params: Vec::new(),
|
||||
defaults: Vec::new(),
|
||||
window: (Timestamp(0), Timestamp(0)),
|
||||
seed: i as u64,
|
||||
broker: "t".to_string(),
|
||||
selection: None,
|
||||
instrument: None,
|
||||
topology_hash: None,
|
||||
project: None,
|
||||
},
|
||||
metrics: RunMetrics {
|
||||
total_pips: v,
|
||||
max_drawdown: v,
|
||||
bias_sign_flips: v as u64,
|
||||
r: None,
|
||||
},
|
||||
},
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn monte_carlo_runs_one_draw_per_seed_in_input_order() {
|
||||
// N seeds -> N draws, seeds carried in INPUT order.
|
||||
let family = monte_carlo(&base_point(), &[1, 2, 3], run_draw);
|
||||
assert_eq!(family.draws.len(), 3);
|
||||
assert_eq!(
|
||||
family.draws.iter().map(|d| d.seed).collect::<Vec<_>>(),
|
||||
vec![1, 2, 3],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn family_is_deterministic_across_thread_counts() {
|
||||
// order = seed input, not completion (C1).
|
||||
let point = base_point();
|
||||
let seeds: Vec<u64> = (1..=8).collect();
|
||||
let one = monte_carlo_with_threads(&point, &seeds, 1, run_draw);
|
||||
let many = monte_carlo_with_threads(&point, &seeds, 8, run_draw);
|
||||
assert_eq!(one, many);
|
||||
assert_eq!(one, monte_carlo(&point, &seeds, run_draw));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn draw_equals_independent_seeded_run() {
|
||||
// each draw == an independent run of that (seed, point)
|
||||
// — MC adds enumeration + execution, never a metrics change (C1).
|
||||
let point = base_point();
|
||||
let family = monte_carlo(&point, &[10, 11, 12], run_draw);
|
||||
for draw in &family.draws {
|
||||
assert_eq!(draw.report, run_draw(draw.seed, &point));
|
||||
assert!(draw.report.metrics.total_pips.is_finite());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distinct_seeds_produce_distinct_draws() {
|
||||
// the seed actually perturbs each run — the family does
|
||||
// not collapse to one metric.
|
||||
let family = monte_carlo(&base_point(), &[1, 2, 3, 4, 5], run_draw);
|
||||
let first = family.draws[0].report.metrics.total_pips;
|
||||
assert!(
|
||||
family.draws.iter().any(|d| d.report.metrics.total_pips != first),
|
||||
"a multi-seed family must not collapse to one metric",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aggregate_is_a_pure_reduction() {
|
||||
// the stored aggregate is recomputable from the draws.
|
||||
let family = monte_carlo(&base_point(), &[1, 2, 3, 4], run_draw);
|
||||
assert_eq!(McAggregate::from_draws(&family.draws), family.aggregate);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aggregate_stats_on_known_fixture() {
|
||||
// type-7 quantile + mean over known metric values
|
||||
// [0,1,2,3,4]: mean=2.0, p50=2.0, p5≈0.2, p95≈3.8.
|
||||
let agg = McAggregate::from_draws(&draws_with_metrics(&[0.0, 1.0, 2.0, 3.0, 4.0]));
|
||||
assert_eq!(agg.total_pips.mean, 2.0);
|
||||
assert_eq!(agg.total_pips.p50, 2.0);
|
||||
assert!((agg.total_pips.p5 - 0.2).abs() < 1e-9, "p5 = {}", agg.total_pips.p5);
|
||||
assert!((agg.total_pips.p95 - 3.8).abs() < 1e-9, "p95 = {}", agg.total_pips.p95);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn r_bootstrap_empty_series_is_all_zero() {
|
||||
let b = r_bootstrap(&[], 100, 1, 7);
|
||||
assert_eq!(b.n_trades, 0);
|
||||
assert_eq!(b.e_r.mean, 0.0);
|
||||
assert_eq!(b.prob_le_zero, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn r_bootstrap_single_block_equals_full_series_mean() {
|
||||
// block_len == n: the only valid start is 0, so every resample IS the full
|
||||
// series -> every resample mean == the series mean -> zero spread.
|
||||
let rs = [1.0, -2.0, 3.0]; // mean = 2/3
|
||||
let b = r_bootstrap(&rs, 500, rs.len(), 7);
|
||||
let mean = 2.0 / 3.0;
|
||||
assert!((b.e_r.mean - mean).abs() < 1e-12);
|
||||
assert!((b.e_r.p5 - mean).abs() < 1e-12);
|
||||
assert!((b.e_r.p95 - mean).abs() < 1e-12);
|
||||
assert_eq!(b.prob_le_zero, 0.0); // mean > 0
|
||||
assert_eq!(b.n_trades, 3);
|
||||
assert_eq!(b.block_len, 3);
|
||||
assert_eq!(b.n_resamples, 500);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn r_bootstrap_is_deterministic_given_seed() {
|
||||
let rs = [0.5, -1.0, 2.0, -0.5, 1.5, -2.0, 0.25];
|
||||
let a = r_bootstrap(&rs, 1000, 1, 42);
|
||||
let b = r_bootstrap(&rs, 1000, 1, 42);
|
||||
assert_eq!(a, b, "same rs+seed+params -> identical RBootstrap (C1)");
|
||||
let c = r_bootstrap(&rs, 1000, 1, 43);
|
||||
assert_ne!(a.e_r.p5, c.e_r.p5, "a different seed reshuffles differently");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn r_bootstrap_block_len_is_clamped_to_series_len() {
|
||||
// block_len > n clamps to n -> single-block behaviour (no panic, no OOB).
|
||||
let rs = [1.0, 2.0];
|
||||
let b = r_bootstrap(&rs, 10, 99, 1);
|
||||
assert_eq!(b.block_len, 2);
|
||||
assert!((b.e_r.mean - 1.5).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn r_bootstrap_moving_block_resamples_are_contiguous_truncated_runs() {
|
||||
// The serial-correlation-preserving branch: 1 < block_len < n with n NOT a
|
||||
// multiple of block_len, so the final block of every resample is truncated
|
||||
// (`take = block_len.min(n - sample.len())`). Every resample is a sequence of
|
||||
// contiguous runs of `rs` (the last one cut short to reach exactly n values),
|
||||
// so each resample mean must lie in the finite set of legal block-composition
|
||||
// means. Distinct powers of ten make every distinct multiset of contributions
|
||||
// a distinct sum -> a faithful membership check. n=5, block_len=2 -> blocks of
|
||||
// length 2, 2, 1.
|
||||
let rs = [1.0, 10.0, 100.0, 1000.0, 10000.0];
|
||||
let n = rs.len();
|
||||
let block_len = 2;
|
||||
|
||||
// Enumerate every legal resample sum: pick a start in [0, n-block_len] for each
|
||||
// of the three blocks (lengths 2, 2, 1), summing min(block_len, n-filled) values
|
||||
// from that start. Membership-by-sum, mirroring r_bootstrap's own construction.
|
||||
let starts: Vec<usize> = (0..=n - block_len).collect();
|
||||
let mut legal_means: Vec<f64> = Vec::new();
|
||||
for &s0 in &starts {
|
||||
for &s1 in &starts {
|
||||
for &s2 in &starts {
|
||||
let mut sum = 0.0;
|
||||
let mut filled = 0usize;
|
||||
for &start in &[s0, s1, s2] {
|
||||
let take = block_len.min(n - filled);
|
||||
if take == 0 {
|
||||
break;
|
||||
}
|
||||
sum += rs[start..start + take].iter().sum::<f64>();
|
||||
filled += take;
|
||||
}
|
||||
legal_means.push(sum / n as f64);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Drive enough resamples that the truncated final block is exercised many
|
||||
// times; assert every produced mean is a legal block-composition mean.
|
||||
let b = r_bootstrap(&rs, 2000, block_len, 7);
|
||||
assert_eq!(b.block_len, 2);
|
||||
assert_eq!(b.n_trades, 5);
|
||||
// The mean of resample means is a weighted average of legal means, so it is
|
||||
// bounded by their min and max (not itself a single legal mean).
|
||||
let lo = legal_means.iter().cloned().fold(f64::INFINITY, f64::min);
|
||||
let hi = legal_means.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
|
||||
assert!(lo - 1e-6 <= b.e_r.mean && b.e_r.mean <= hi + 1e-6);
|
||||
// Each quantile of the resampled E[R] is an order statistic of the resample
|
||||
// means, hence itself a legal contiguous-block-composition mean.
|
||||
for &m in &[b.e_r.p5, b.e_r.p25, b.e_r.p50, b.e_r.p75, b.e_r.p95] {
|
||||
assert!(
|
||||
legal_means.iter().any(|&lm| (lm - m).abs() < 1e-6),
|
||||
"resampled quantile {m} is not a legal contiguous-block-composition mean",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,34 @@
|
||||
//! Every type's serde shape is byte-pinned by C18 goldens; no float expression
|
||||
//! re-associated.
|
||||
|
||||
use aura_core::{Scalar, Timestamp};
|
||||
use aura_core::{Scalar, SeriesFold, Timestamp};
|
||||
|
||||
/// Reduce a run's recorded pip-equity + exposure streams into summary metrics.
|
||||
/// Pure — identical inputs yield identical metrics (C1/C12). Timestamps are
|
||||
/// carried in the input to match exactly what a sink records; the reduction
|
||||
/// itself is value-only (it does not read the timestamps).
|
||||
pub fn summarize(
|
||||
equity: &[(Timestamp, f64)],
|
||||
exposure: &[(Timestamp, f64)],
|
||||
) -> RunMetrics {
|
||||
// total_pips + max_drawdown fold over equity; sign_flips folds over exposure.
|
||||
// SeriesFold is the single arithmetic source of truth (shared with the
|
||||
// SeriesReducer sink), so this is byte-identical to the prior inline loops.
|
||||
let mut eqf = SeriesFold::new();
|
||||
for &(_, v) in equity {
|
||||
eqf.fold(v);
|
||||
}
|
||||
let mut exf = SeriesFold::new();
|
||||
for &(_, v) in exposure {
|
||||
exf.fold(v);
|
||||
}
|
||||
RunMetrics {
|
||||
total_pips: eqf.last,
|
||||
max_drawdown: eqf.max_drawdown,
|
||||
bias_sign_flips: exf.sign_flips,
|
||||
r: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Summary metrics reduced from a run's recorded streams — the `-> metrics`
|
||||
/// half of C12's atomic sim unit. Pure function of the recorded streams.
|
||||
@@ -1048,4 +1075,181 @@ mod tests {
|
||||
assert_eq!(m.expectancy_r, 0.0);
|
||||
assert_eq!(m.sqn, 0.0);
|
||||
}
|
||||
|
||||
// The `summarize` end-to-end + unit tests, moved verbatim from
|
||||
// `aura-engine::report` (#291, C28 phase 2): `run_once` drives a real
|
||||
// bootstrapped Harness through public aura-engine/aura-std/aura-strategy
|
||||
// APIs, so it needs no engine-private item.
|
||||
use aura_core::{Firing, NodeSchema, PortSpec, ScalarKind};
|
||||
use aura_engine::{f64_field, Edge, FlatGraph, Harness, RunManifest, RunReport, SourceSpec, Target, VecSource};
|
||||
use aura_std::{Recorder, Sma, Sub};
|
||||
use aura_strategy::Bias;
|
||||
use crate::SimBroker;
|
||||
use std::sync::mpsc;
|
||||
|
||||
/// The declared signature of a `Recorder` over one f64 column (the sink shape
|
||||
/// the two-sink harness uses).
|
||||
fn f64_recorder_sig() -> NodeSchema {
|
||||
NodeSchema {
|
||||
inputs: vec![PortSpec { kind: ScalarKind::F64, firing: Firing::Any, name: "in".into() }],
|
||||
output: vec![],
|
||||
params: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
/// Build an f64 source stream from (timestamp, value) points (mirrors the
|
||||
/// harness.rs test helper; the e2e test needs its own copy — the harness
|
||||
/// test module's is private to that module).
|
||||
fn f64_stream(points: &[(i64, f64)]) -> Vec<(Timestamp, Scalar)> {
|
||||
points.iter().map(|&(t, v)| (Timestamp(t), Scalar::f64(v))).collect()
|
||||
}
|
||||
|
||||
/// Bootstrap the cycle-0007 signal-quality harness with TWO sinks: one on
|
||||
/// the SimBroker equity output (node 4 -> node 5) and one on the Bias
|
||||
/// output (node 3 -> node 6). Returns the harness plus the two receivers.
|
||||
#[allow(clippy::type_complexity)]
|
||||
fn build_two_sink_harness() -> (
|
||||
Harness,
|
||||
mpsc::Receiver<(Timestamp, Vec<Scalar>)>,
|
||||
mpsc::Receiver<(Timestamp, Vec<Scalar>)>,
|
||||
) {
|
||||
let (tx_eq, rx_eq) = mpsc::channel();
|
||||
let (tx_ex, rx_ex) = mpsc::channel();
|
||||
let h = Harness::bootstrap(FlatGraph {
|
||||
nodes: vec![
|
||||
Box::new(Sma::new(2)), // 0
|
||||
Box::new(Sma::new(4)), // 1
|
||||
Box::new(Sub::new()), // 2
|
||||
Box::new(Bias::new(0.5)), // 3
|
||||
Box::new(SimBroker::new(0.0001)), // 4
|
||||
Box::new(Recorder::new(&[ScalarKind::F64], Firing::Any, tx_eq)), // 5 equity sink
|
||||
Box::new(Recorder::new(&[ScalarKind::F64], Firing::Any, tx_ex)), // 6 exposure sink
|
||||
],
|
||||
signatures: vec![
|
||||
Sma::builder().schema().clone(),
|
||||
Sma::builder().schema().clone(),
|
||||
Sub::builder().schema().clone(),
|
||||
Bias::builder().schema().clone(),
|
||||
SimBroker::builder(0.0001).schema().clone(),
|
||||
f64_recorder_sig(),
|
||||
f64_recorder_sig(),
|
||||
],
|
||||
sources: vec![SourceSpec::raw(ScalarKind::F64, vec![
|
||||
Target { node: 0, slot: 0 },
|
||||
Target { node: 1, slot: 0 },
|
||||
Target { node: 4, slot: 1 }, // price into the broker
|
||||
])],
|
||||
edges: vec![
|
||||
Edge { from: 0, to: 2, slot: 0, from_field: 0 },
|
||||
Edge { from: 1, to: 2, slot: 1, from_field: 0 },
|
||||
Edge { from: 2, to: 3, slot: 0, from_field: 0 },
|
||||
Edge { from: 3, to: 4, slot: 0, from_field: 0 },
|
||||
Edge { from: 4, to: 5, slot: 0, from_field: 0 }, // equity -> sink 5
|
||||
Edge { from: 3, to: 6, slot: 0, from_field: 0 }, // exposure -> sink 6
|
||||
],
|
||||
taps: Vec::new(),
|
||||
})
|
||||
.expect("valid signal-quality DAG");
|
||||
(h, rx_eq, rx_ex)
|
||||
}
|
||||
|
||||
fn run_once() -> RunReport<RunMetrics> {
|
||||
let (mut h, rx_eq, rx_ex) = build_two_sink_harness();
|
||||
h.run(vec![Box::new(VecSource::new(f64_stream(&[
|
||||
(1, 1.0000),
|
||||
(2, 1.0010),
|
||||
(3, 1.0025),
|
||||
(4, 1.0020),
|
||||
(5, 1.0040),
|
||||
])))]);
|
||||
let eq_rows: Vec<(Timestamp, Vec<Scalar>)> = rx_eq.try_iter().collect();
|
||||
let ex_rows: Vec<(Timestamp, Vec<Scalar>)> = rx_ex.try_iter().collect();
|
||||
let equity = f64_field(&eq_rows, 0);
|
||||
let exposure = f64_field(&ex_rows, 0);
|
||||
let metrics = summarize(&equity, &exposure);
|
||||
RunReport {
|
||||
manifest: RunManifest {
|
||||
commit: "test-commit".to_string(),
|
||||
params: vec![
|
||||
("sma_fast".to_string(), Scalar::i64(2)),
|
||||
("sma_slow".to_string(), Scalar::i64(4)),
|
||||
("bias_scale".to_string(), Scalar::f64(0.5)),
|
||||
],
|
||||
defaults: vec![],
|
||||
window: (Timestamp(1), Timestamp(5)),
|
||||
seed: 0,
|
||||
broker: "sim-optimal(pip_size=0.0001)".to_string(),
|
||||
selection: None,
|
||||
instrument: None,
|
||||
topology_hash: None,
|
||||
project: None,
|
||||
},
|
||||
metrics,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn report_is_deterministic_end_to_end() {
|
||||
let r1 = run_once();
|
||||
let r2 = run_once();
|
||||
// a run actually emitted metrics over a non-empty pip curve
|
||||
assert!(r1.metrics.total_pips.is_finite());
|
||||
// same manifest -> same metrics (C1/C12): two runs are bit-identical
|
||||
assert_eq!(r1.metrics, r2.metrics);
|
||||
assert_eq!(r1.to_json(), r2.to_json());
|
||||
}
|
||||
|
||||
fn samples(values: &[f64]) -> Vec<(Timestamp, f64)> {
|
||||
values
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, &v)| (Timestamp(i as i64 + 1), v))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_total_pips_is_last_cumulative_value() {
|
||||
let equity = samples(&[0.0, 5.0, 4.0, 12.0]);
|
||||
let m = summarize(&equity, &[]);
|
||||
assert_eq!(m.total_pips, 12.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_is_zero_on_empty_streams() {
|
||||
let m = summarize(&[], &[]);
|
||||
assert_eq!(m.total_pips, 0.0);
|
||||
assert_eq!(m.max_drawdown, 0.0);
|
||||
assert_eq!(m.bias_sign_flips, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_max_drawdown_is_worst_peak_to_trough() {
|
||||
// peak 10 then trough 5 (drop 5), recovers to 8; worst drop is 5,
|
||||
// not the final drop (10 -> 8 = 2).
|
||||
let equity = samples(&[0.0, 10.0, 5.0, 8.0]);
|
||||
let m = summarize(&equity, &[]);
|
||||
assert_eq!(m.max_drawdown, 5.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_max_drawdown_zero_on_monotonic_curve() {
|
||||
let equity = samples(&[0.0, 1.0, 2.0, 3.0]);
|
||||
let m = summarize(&equity, &[]);
|
||||
assert_eq!(m.max_drawdown, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_sign_flips_counts_signum_changes() {
|
||||
// signum series: + + - 0 - -> flips at +->-, -->0, 0->- = 3.
|
||||
let exposure = samples(&[0.5, 0.5, -0.5, 0.0, -0.5]);
|
||||
let m = summarize(&[], &exposure);
|
||||
assert_eq!(m.bias_sign_flips, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarize_sign_flips_zero_on_constant_sign() {
|
||||
let exposure = samples(&[0.2, 0.5, 1.0, 0.7]);
|
||||
let m = summarize(&[], &exposure);
|
||||
assert_eq!(m.bias_sign_flips, 0);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user