feat(analysis,campaign,registry,cli): bootstrap net R through the process pipeline
The OOS bootstrap conduit RMetrics.trade_rs becomes net_trade_rs and carries the COST-NETTED per-trade R (r − cost_in_r, trade order). Every conduit consumer is thereby net-when-costed: the monte-carlo pooled-OOS and per-survivor bootstraps, the walk-forward oos_r pooling, and the deflation null-max — which previously compared a net observed statistic against a gross-resampled null whenever a costed campaign selected on net_expectancy_r. An uncosted run is bit-identical (empty cost stream ⇒ cost 0.0 per trade), so every existing golden pin stays green. Design: one conduit, no knob — an explicit net: knob would let a costed campaign silently produce a gross headline again (the exact misreading of the issue's evidence). Per-member RMetrics scalar fields stay gross; net_expectancy_r keeps its meaning. Wire shape unchanged (serde(skip)). The net series is materialized as a separate expression in summarize_r; the pinned-float expressions (net_sum, the SQN pair, the lockstep r_metrics_from_rs copies) keep their tokens verbatim. Fork decisions and rationales: issue #259 comments. New coverage, both hostless over the synthetic SYMA archive: a costed campaign twin shifts the pooled-OOS bootstrap (sweep→gate→wf→mc) and every per-survivor bootstrap (sweep→mc) below its gross sibling, with the trade population unchanged; a unit test pins the conduit as the cost-netted series with the empty-stream degeneracy. Existing conduit tests renamed with the field. Verified: full workspace suite green (71 result blocks), clippy clean, zero bare trade_rs tokens remain, ledger conduit prose updated in place. closes #259
This commit is contained in:
@@ -875,11 +875,11 @@ fn select_winner(
|
||||
/// (window order, then within-window trade order). Windows with no `r` block
|
||||
/// contribute nothing. The single home of the pooling-in-roll-order semantics —
|
||||
/// both the walk-forward `oos_r` summary and the `mc` R-bootstrap reduce this.
|
||||
fn pooled_oos_trade_rs(result: &WalkForwardResult) -> Vec<f64> {
|
||||
fn pooled_oos_net_trade_rs(result: &WalkForwardResult) -> Vec<f64> {
|
||||
result
|
||||
.windows
|
||||
.iter()
|
||||
.flat_map(|w| w.run.oos_report.metrics.r.as_ref().map(|r| r.trade_rs.clone()).unwrap_or_default())
|
||||
.flat_map(|w| w.run.oos_report.metrics.r.as_ref().map(|r| r.net_trade_rs.clone()).unwrap_or_default())
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -888,14 +888,14 @@ fn pooled_oos_trade_rs(result: &WalkForwardResult) -> Vec<f64> {
|
||||
/// (C14).
|
||||
fn walkforward_summary_json(result: &WalkForwardResult) -> String {
|
||||
let total = result.stitched_oos_equity.last().map(|&(_, v)| v).unwrap_or(0.0);
|
||||
let pooled_rs = pooled_oos_trade_rs(result);
|
||||
let pooled_rs = pooled_oos_net_trade_rs(result);
|
||||
let mut obj = serde_json::json!({
|
||||
"windows": result.windows.len(),
|
||||
"stitched_total_pips": total,
|
||||
"param_stability": param_stability(result),
|
||||
});
|
||||
if result.windows.iter().any(|w| w.run.oos_report.metrics.r.is_some()) {
|
||||
// RMetrics serializes its scalar fields (trade_rs is serde-skipped, so the
|
||||
// RMetrics serializes its scalar fields (net_trade_rs is serde-skipped, so the
|
||||
// oos_r block is the clean R-metric summary of the pooled series).
|
||||
obj["oos_r"] = serde_json::to_value(r_metrics_from_rs(&pooled_rs))
|
||||
.expect("RMetrics serializes");
|
||||
@@ -914,7 +914,7 @@ fn walkforward_summary_json(result: &WalkForwardResult) -> String {
|
||||
/// (e.g. `sma_signal.fast.length`) by the strategy's own param space, while
|
||||
/// `stop_length`/`stop_k` ride the risk regime unwrapped, so an exact-name match
|
||||
/// would miss the wrapped ones) through the same `MetricStats::from_values`; the
|
||||
/// `oos_r` block pools the per-window `trade_rs` through `r_metrics_from_rs`.
|
||||
/// `oos_r` block pools the per-window `net_trade_rs` through `r_metrics_from_rs`.
|
||||
/// Canonical JSON (C14).
|
||||
///
|
||||
/// `axes` carries the invocation's raw axis names in argv order, followed by the
|
||||
@@ -954,7 +954,7 @@ fn walkforward_summary_json_from_reports(reports: &[RunReport], axes: &[String])
|
||||
.collect();
|
||||
let pooled_rs: Vec<f64> = reports
|
||||
.iter()
|
||||
.flat_map(|r| r.metrics.r.as_ref().map(|m| m.trade_rs.clone()).unwrap_or_default())
|
||||
.flat_map(|r| r.metrics.r.as_ref().map(|m| m.net_trade_rs.clone()).unwrap_or_default())
|
||||
.collect();
|
||||
let mut obj = serde_json::json!({
|
||||
"windows": reports.len(),
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
mod common;
|
||||
use common::{ScratchGuard, ScratchPath, fresh_project};
|
||||
use common::{ScratchGuard, ScratchPath, fresh_project, fresh_project_with_data};
|
||||
|
||||
/// A fresh, unique working directory for a process test that persists
|
||||
/// content-addressed documents under `./runs/` (so `aura process register`
|
||||
@@ -1010,7 +1010,10 @@ fn register_process_doc(dir: &Path, file: &str, doc: &str) -> String {
|
||||
/// the RAW `param_space` names (`fast.length` / `slow.length` — see the
|
||||
/// naming note in the referential test above). `persist_taps`/`emit` are
|
||||
/// spliced verbatim (pass `""` for empty, `"\"family_table\""` etc.).
|
||||
fn campaign_doc_json(
|
||||
/// [`campaign_doc_json`] with the instrument spliced — the synthetic-archive
|
||||
/// e2e targets `SYMA` (tests/common/mod.rs) instead of the hardcoded GER40.
|
||||
fn campaign_doc_json_for(
|
||||
instrument: &str,
|
||||
bp_id: &str,
|
||||
proc_id: &str,
|
||||
window: (i64, i64),
|
||||
@@ -1022,7 +1025,7 @@ fn campaign_doc_json(
|
||||
"format_version": 1,
|
||||
"kind": "campaign",
|
||||
"name": "run-seam",
|
||||
"data": {{ "instruments": ["GER40"], "windows": [ {{ "from_ms": {from}, "to_ms": {to} }} ] }},
|
||||
"data": {{ "instruments": ["{instrument}"], "windows": [ {{ "from_ms": {from}, "to_ms": {to} }} ] }},
|
||||
"strategies": [ {{ "ref": {{ "content_id": "{bp_id}" }},
|
||||
"axes": {{ "fast.length": {{ "kind": "I64", "values": [2, 4] }},
|
||||
"slow.length": {{ "kind": "I64", "values": [8, 16] }} }} }} ],
|
||||
@@ -1035,6 +1038,16 @@ fn campaign_doc_json(
|
||||
)
|
||||
}
|
||||
|
||||
fn campaign_doc_json(
|
||||
bp_id: &str,
|
||||
proc_id: &str,
|
||||
window: (i64, i64),
|
||||
persist_taps: &str,
|
||||
emit: &str,
|
||||
) -> String {
|
||||
campaign_doc_json_for("GER40", bp_id, proc_id, window, persist_taps, emit)
|
||||
}
|
||||
|
||||
/// An mc-bearing process: intrinsically valid AND (since the v2 executor) an
|
||||
/// executable pipeline shape — `sweep -> monte_carlo` annotates each sweep
|
||||
/// survivor. The two addressing-mode tests below run it over the [1, 2] 1970
|
||||
@@ -1990,6 +2003,163 @@ fn campaign_run_real_e2e_sweep_gate_walkforward() {
|
||||
);
|
||||
}
|
||||
|
||||
/// Property (#259): a campaign's cost block reaches the walk-forward →
|
||||
/// monte-carlo evidence chain — the pooled-OOS bootstrap resamples the
|
||||
/// COST-NETTED per-trade series, so the same matrix run with and without a
|
||||
/// cost block records DIFFERENT `pooled_oos` stats (today's bug: identical to
|
||||
/// every digit). Hostless: runs over the synthetic SYMA archive, no
|
||||
/// data-mount skip-guard.
|
||||
#[test]
|
||||
fn campaign_run_synthetic_e2e_cost_block_nets_the_pooled_oos_bootstrap() {
|
||||
let (dir, _fixture) = fresh_project_with_data();
|
||||
let runs_dir = dir.join("runs");
|
||||
std::fs::remove_dir_all(&runs_dir).ok();
|
||||
let _cleanup = ScratchGuard(vec![
|
||||
ScratchPath::Dir(runs_dir.clone()),
|
||||
ScratchPath::File(dir.join("netmc.process.json")),
|
||||
ScratchPath::File(dir.join("gross.campaign.json")),
|
||||
ScratchPath::File(dir.join("net.campaign.json")),
|
||||
]);
|
||||
let bp_id = seed_blueprint(&dir, "campaign-run-netmc-seed");
|
||||
let proc_id = register_process_doc(&dir, "netmc.process.json", WF_PROCESS_DOC);
|
||||
// SYMA's synthetic archive spans 2024-01..08; Mar-01..Jun-30 (~17 weeks)
|
||||
// gives the (14d, 7d, 7d) roller a comfortable tiling.
|
||||
let base = campaign_doc_json_for(
|
||||
"SYMA",
|
||||
&bp_id,
|
||||
&proc_id,
|
||||
(1709251200000, 1719791999999),
|
||||
"",
|
||||
"",
|
||||
);
|
||||
// The costed twin: the SAME document plus a constant cost block — the
|
||||
// only difference (the replacen pattern of the cost e2e siblings).
|
||||
let with_cost = base.replacen(
|
||||
"\"seed\": 7,",
|
||||
"\"seed\": 7,\n \"cost\": [ { \"constant\": { \"cost_per_trade\": 0.5 } } ],",
|
||||
1,
|
||||
);
|
||||
assert_ne!(with_cost, base, "replacen must actually match the seed field");
|
||||
write_doc(&dir, "gross.campaign.json", &base);
|
||||
write_doc(&dir, "net.campaign.json", &with_cost);
|
||||
|
||||
// Both runs share the store (the cost-e2e sibling pattern) — the record
|
||||
// line is read from each run's own stdout, so no isolation is needed.
|
||||
let pooled = |doc: &str| -> serde_json::Value {
|
||||
let (out, code) = run_code_in(&dir, &["campaign", "run", doc]);
|
||||
assert_eq!(code, Some(0), "campaign run failed: {out}");
|
||||
let line = out
|
||||
.lines()
|
||||
.find(|l| l.starts_with("{\"campaign_run\":"))
|
||||
.expect("the always-on final campaign_run line");
|
||||
let v: serde_json::Value = serde_json::from_str(line).expect("record parses");
|
||||
v["campaign_run"]["cells"][0]["stages"][3]["bootstrap"]["pooled_oos"].clone()
|
||||
};
|
||||
let gross = pooled("gross.campaign.json");
|
||||
let net = pooled("net.campaign.json");
|
||||
|
||||
let n_gross = gross["n_trades"].as_u64().expect("gross n_trades");
|
||||
let n_net = net["n_trades"].as_u64().expect("net n_trades");
|
||||
assert!(n_gross > 0, "the synthetic window must produce OOS trades: {gross}");
|
||||
assert_eq!(
|
||||
n_gross, n_net,
|
||||
"cost is a feed-forward subtraction — it must not change the trade population"
|
||||
);
|
||||
let mean_gross = gross["e_r"]["mean"].as_f64().expect("gross mean");
|
||||
let mean_net = net["e_r"]["mean"].as_f64().expect("net mean");
|
||||
assert!(
|
||||
mean_net < mean_gross,
|
||||
"the costed bootstrap must resample the NET series (#259): gross {mean_gross} vs net {mean_net}"
|
||||
);
|
||||
let p_gross = gross["prob_le_zero"].as_f64().expect("gross prob_le_zero");
|
||||
let p_net = net["prob_le_zero"].as_f64().expect("net prob_le_zero");
|
||||
assert!(
|
||||
p_net >= p_gross,
|
||||
"a strictly-lower resample mean cannot lower prob_le_zero: gross {p_gross} vs net {p_net}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Property (#259), the OTHER mc bootstrap input shape: with no
|
||||
/// `walk_forward` in the pipeline (`sweep -> monte_carlo`), each surviving
|
||||
/// sweep member gets its OWN bootstrap (`PerSurvivor`, keyed by ordinal) —
|
||||
/// a distinct code path from `PooledOos` above (`member_net_trade_rs` /
|
||||
/// `SweepPoint` in aura-registry, not the wf-family pooling in
|
||||
/// aura-campaign). That path must resample the same cost-netted
|
||||
/// `net_trade_rs` conduit: a cost block must shift EVERY surviving member's
|
||||
/// resampled mean down, never leave any member's bootstrap gross-identical.
|
||||
/// Hostless: runs over the synthetic SYMA archive, no data-mount skip-guard.
|
||||
/// The gated real-data sibling (`campaign_run_real_e2e_sweep_monte_carlo_per_survivor`)
|
||||
/// exercises this shape but never with a cost block, so it cannot pin this property.
|
||||
#[test]
|
||||
fn campaign_run_synthetic_e2e_cost_block_nets_the_per_survivor_bootstrap() {
|
||||
let (dir, _fixture) = fresh_project_with_data();
|
||||
let runs_dir = dir.join("runs");
|
||||
std::fs::remove_dir_all(&runs_dir).ok();
|
||||
let _cleanup = ScratchGuard(vec![
|
||||
ScratchPath::Dir(runs_dir.clone()),
|
||||
ScratchPath::File(dir.join("persurvivor.process.json")),
|
||||
ScratchPath::File(dir.join("gross_ps.campaign.json")),
|
||||
ScratchPath::File(dir.join("net_ps.campaign.json")),
|
||||
]);
|
||||
let bp_id = seed_blueprint(&dir, "campaign-run-persurvivor-seed");
|
||||
let proc_id = register_process_doc(&dir, "persurvivor.process.json", MC_PROCESS_DOC);
|
||||
// Same SYMA window as the pooled-OOS sibling above — known to produce
|
||||
// trades over the synthetic archive.
|
||||
let base = campaign_doc_json_for(
|
||||
"SYMA",
|
||||
&bp_id,
|
||||
&proc_id,
|
||||
(1709251200000, 1719791999999),
|
||||
"",
|
||||
"",
|
||||
);
|
||||
let with_cost = base.replacen(
|
||||
"\"seed\": 7,",
|
||||
"\"seed\": 7,\n \"cost\": [ { \"constant\": { \"cost_per_trade\": 0.5 } } ],",
|
||||
1,
|
||||
);
|
||||
assert_ne!(with_cost, base, "replacen must actually match the seed field");
|
||||
write_doc(&dir, "gross_ps.campaign.json", &base);
|
||||
write_doc(&dir, "net_ps.campaign.json", &with_cost);
|
||||
|
||||
let per_survivor = |doc: &str| -> Vec<(u64, serde_json::Value)> {
|
||||
let (out, code) = run_code_in(&dir, &["campaign", "run", doc]);
|
||||
assert_eq!(code, Some(0), "campaign run failed: {out}");
|
||||
let line = out
|
||||
.lines()
|
||||
.find(|l| l.starts_with("{\"campaign_run\":"))
|
||||
.expect("the always-on final campaign_run line");
|
||||
let v: serde_json::Value = serde_json::from_str(line).expect("record parses");
|
||||
v["campaign_run"]["cells"][0]["stages"][1]["bootstrap"]["per_survivor"]
|
||||
.as_array()
|
||||
.expect("no walk_forward precedes: the PerSurvivor arm, not pooled_oos")
|
||||
.iter()
|
||||
.map(|pair| (pair[0].as_u64().expect("ordinal"), pair[1].clone()))
|
||||
.collect()
|
||||
};
|
||||
let gross = per_survivor("gross_ps.campaign.json");
|
||||
let net = per_survivor("net_ps.campaign.json");
|
||||
|
||||
assert_eq!(gross.len(), 4, "2x2 axes -> four surviving members: {gross:?}");
|
||||
assert_eq!(gross.len(), net.len(), "cost must not change the surviving-member population");
|
||||
for ((g_ord, g), (n_ord, n)) in gross.iter().zip(net.iter()) {
|
||||
assert_eq!(g_ord, n_ord, "the two runs must enumerate members in the same ordinal order");
|
||||
let n_trades_g = g["n_trades"].as_u64().expect("gross n_trades");
|
||||
let n_trades_n = n["n_trades"].as_u64().expect("net n_trades");
|
||||
assert!(n_trades_g > 0, "member {g_ord} must have produced trades: {g}");
|
||||
assert_eq!(
|
||||
n_trades_g, n_trades_n,
|
||||
"member {g_ord}: cost is a feed-forward subtraction — it must not change the trade population"
|
||||
);
|
||||
let mean_g = g["e_r"]["mean"].as_f64().expect("gross mean");
|
||||
let mean_n = n["e_r"]["mean"].as_f64().expect("net mean");
|
||||
assert!(
|
||||
mean_n < mean_g,
|
||||
"member {g_ord}: the costed PerSurvivor bootstrap must resample the NET series (#259): gross {mean_g} vs net {mean_n}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The campaign `data.bindings` override end to end (#231, 6b): the same
|
||||
/// seeded strategy, window, and seed run twice — bare (price<-close default)
|
||||
/// and with `price` rebound to the open column. Both exit 0; the realized
|
||||
|
||||
Reference in New Issue
Block a user