From 0408fc41a2791b8600bcd86c7878f90c52ac42f3 Mon Sep 17 00:00:00 2001 From: Brummel Date: Fri, 10 Jul 2026 14:47:23 +0200 Subject: [PATCH] =?UTF-8?q?test(cli):=20RED=20=E2=80=94=20reproduce=20over?= =?UTF-8?q?=20real-data=20families=20diverges/panics=20(#229)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two failing e2e pins (skip-on-no-data, GER40 archive present here): - reproduce_real_sweep_family_re_derives_bit_identically: a --real sweep family must reproduce N/N bit-identically; currently 0/N DIVERGED because reproduce_family re-runs members over DataSource::Synthetic. - reproduce_real_walkforward_family_does_not_panic: a --real WalkForward family must yield a verdict or a clean refusal; currently panics at the empty-OOS-window expect (main.rs:1123) for the same reason. refs #229 --- crates/aura-cli/tests/cli_run.rs | 125 +++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/crates/aura-cli/tests/cli_run.rs b/crates/aura-cli/tests/cli_run.rs index 0d9582a..d1bfec7 100644 --- a/crates/aura-cli/tests/cli_run.rs +++ b/crates/aura-cli/tests/cli_run.rs @@ -1225,6 +1225,131 @@ fn sweep_real_blueprint_member_lines_pin_the_dissolved_contract() { assert_eq!(count("campaigns"), 1, "second identical run dedupes the campaign doc"); } +/// Extract the first persisted family whose `kind` matches `want` from +/// `aura runs families` stdout (one JSON object per line), returning its +/// `family_id` — the id `aura reproduce` addresses. Panics if none matches so a +/// mis-minted store fails loudly rather than skipping the assertion. +fn family_id_of_kind(dir: &Path, want: &str) -> String { + let out = Command::new(BIN).args(["runs", "families"]).current_dir(dir).output().unwrap(); + assert!(out.status.success(), "runs families exit: {:?}", out.status); + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + for line in stdout.lines() { + let v: serde_json::Value = serde_json::from_str(line).expect("family line parses as JSON"); + if v["kind"].as_str() == Some(want) { + return v["family_id"].as_str().expect("family_id is a string").to_string(); + } + } + panic!("no {want} family in the store: {stdout}"); +} + +/// Property (C18/C1 reproduce guarantee): `aura reproduce` re-derives a family +/// over the SAME data it was minted on. A sweep family minted over real GER40 +/// bars must therefore reproduce bit-identically — the run is deterministic +/// (a fresh identical sweep matches the persisted metrics to the last digit), so +/// the README's "check it is bit-identical" promise must hold. It must NOT +/// silently re-run every member over the built-in synthetic showcase stream, +/// which diverges from the stored real-data metrics on every member (observed: +/// `reproduced 0/N bit-identically`, exit 1). Gated on the local GER40 archive +/// (skip-on-no-data convention); uses the plain `r_sma_open.json` blueprint — +/// no gang, no campaign-specific fixture — because the divergence is driven by +/// `--real`, not by the minting path. +#[test] +fn reproduce_real_sweep_family_re_derives_bit_identically() { + if !local_data_present() { + eprintln!("skip: no local data at {}", data_server::DEFAULT_DATA_PATH); + return; + } + let (dir, _g) = fresh_project(); + let fixture = format!("{}/examples/r_sma_open.json", env!("CARGO_MANIFEST_DIR")); + let mint = Command::new(BIN) + .args([ + "sweep", &fixture, + "--real", "GER40", + "--from", GER40_SEPT2024_FROM_MS, + "--to", GER40_SEPT2024_TO_MS, + "--axis", "sma_signal.fast.length=2,4", + "--axis", "sma_signal.slow.length=8", + "--name", "repro_sweep", + ]) + .current_dir(dir) + .output() + .unwrap(); + assert!(mint.status.success(), "sweep mint stderr: {}", String::from_utf8_lossy(&mint.stderr)); + + let id = family_id_of_kind(dir, "Sweep"); + let out = Command::new(BIN).args(["reproduce", &id]).current_dir(dir).output().unwrap(); + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + assert!( + out.status.success(), + "reproduce of a real-data sweep family must succeed (exit 0), got {:?}\nstdout: {stdout}\nstderr: {}", + out.status.code(), + String::from_utf8_lossy(&out.stderr) + ); + assert!( + stdout.contains("reproduced 2/2 members bit-identically"), + "every member of a deterministic real-data sweep re-derives bit-identically: {stdout}" + ); + assert!( + !stdout.contains("DIVERGED"), + "no member may DIVERGE — the run is deterministic, reproduce must use the recorded data source: {stdout}" + ); +} + +/// Property: `aura reproduce` yields a handled outcome — a reproduction verdict +/// or a clean `aura:` refusal — for a real-data WalkForward family, and NEVER an +/// internal panic. A WalkForward member's OOS window is stored as real epoch-ns +/// bounds; reconstructing that window over the built-in synthetic walk (whose 60 +/// bars fall entirely outside those bounds) yields an empty source, which today +/// reaches the unguarded `window_of(&s).expect("non-empty OOS window")` and +/// aborts the process (exit 101). Gated on the local GER40 archive +/// (skip-on-no-data convention); a ~135-day window is the minimum that stitches +/// at least one OOS window (IS 90d + OOS 30d). +#[test] +fn reproduce_real_walkforward_family_does_not_panic() { + if !local_data_present() { + eprintln!("skip: no local data at {}", data_server::DEFAULT_DATA_PATH); + return; + } + // 2025-01-01 .. ~2025-05-16 (~135 days) — long enough for one IS(90d)+OOS(30d) + // walk-forward window over the local GER40 archive. + const WF_FROM_MS: &str = "1735689600000"; + const WF_TO_MS: &str = "1747353600000"; + let (dir, _g) = fresh_project(); + let fixture = format!("{}/examples/r_sma_open.json", env!("CARGO_MANIFEST_DIR")); + let mint = Command::new(BIN) + .args([ + "walkforward", &fixture, + "--real", "GER40", + "--from", WF_FROM_MS, + "--to", WF_TO_MS, + "--axis", "sma_signal.fast.length=2,4", + "--axis", "sma_signal.slow.length=8", + "--name", "repro_wf", + ]) + .current_dir(dir) + .output() + .unwrap(); + assert!(mint.status.success(), "walkforward mint stderr: {}", String::from_utf8_lossy(&mint.stderr)); + + let id = family_id_of_kind(dir, "WalkForward"); + let out = Command::new(BIN).args(["reproduce", &id]).current_dir(dir).output().unwrap(); + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); + assert_ne!( + out.status.code(), + Some(101), + "reproduce of a WalkForward family must not panic (exit 101)\nstdout: {stdout}\nstderr: {stderr}" + ); + assert!( + !stderr.contains("panicked") && !stderr.contains("non-empty OOS window"), + "no internal panic may leak from reproduce: {stderr}" + ); + assert!( + stdout.contains("reproduced") || stderr.starts_with("aura:"), + "reproduce must yield a verdict (stdout) or a clean `aura:` refusal (stderr): stdout={stdout} stderr={stderr}" + ); +} + /// Property (#210 c0110 fieldtest, "the wrapped-name refusal mangles the axis /// the user typed"): the dissolved real-data blueprint sweep types `--axis` /// against the WRAPPED probe namespace (`sma_signal.fast.length`, what