test(cli): RED — reproduce over real-data families diverges/panics (#229)
Two failing e2e pins (skip-on-no-data, GER40 archive present here): - reproduce_real_sweep_family_re_derives_bit_identically: a --real sweep family must reproduce N/N bit-identically; currently 0/N DIVERGED because reproduce_family re-runs members over DataSource::Synthetic. - reproduce_real_walkforward_family_does_not_panic: a --real WalkForward family must yield a verdict or a clean refusal; currently panics at the empty-OOS-window expect (main.rs:1123) for the same reason. refs #229
This commit is contained in:
@@ -1225,6 +1225,131 @@ fn sweep_real_blueprint_member_lines_pin_the_dissolved_contract() {
|
|||||||
assert_eq!(count("campaigns"), 1, "second identical run dedupes the campaign doc");
|
assert_eq!(count("campaigns"), 1, "second identical run dedupes the campaign doc");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Extract the first persisted family whose `kind` matches `want` from
|
||||||
|
/// `aura runs families` stdout (one JSON object per line), returning its
|
||||||
|
/// `family_id` — the id `aura reproduce` addresses. Panics if none matches so a
|
||||||
|
/// mis-minted store fails loudly rather than skipping the assertion.
|
||||||
|
fn family_id_of_kind(dir: &Path, want: &str) -> String {
|
||||||
|
let out = Command::new(BIN).args(["runs", "families"]).current_dir(dir).output().unwrap();
|
||||||
|
assert!(out.status.success(), "runs families exit: {:?}", out.status);
|
||||||
|
let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
|
||||||
|
for line in stdout.lines() {
|
||||||
|
let v: serde_json::Value = serde_json::from_str(line).expect("family line parses as JSON");
|
||||||
|
if v["kind"].as_str() == Some(want) {
|
||||||
|
return v["family_id"].as_str().expect("family_id is a string").to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
panic!("no {want} family in the store: {stdout}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Property (C18/C1 reproduce guarantee): `aura reproduce` re-derives a family
|
||||||
|
/// over the SAME data it was minted on. A sweep family minted over real GER40
|
||||||
|
/// bars must therefore reproduce bit-identically — the run is deterministic
|
||||||
|
/// (a fresh identical sweep matches the persisted metrics to the last digit), so
|
||||||
|
/// the README's "check it is bit-identical" promise must hold. It must NOT
|
||||||
|
/// silently re-run every member over the built-in synthetic showcase stream,
|
||||||
|
/// which diverges from the stored real-data metrics on every member (observed:
|
||||||
|
/// `reproduced 0/N bit-identically`, exit 1). Gated on the local GER40 archive
|
||||||
|
/// (skip-on-no-data convention); uses the plain `r_sma_open.json` blueprint —
|
||||||
|
/// no gang, no campaign-specific fixture — because the divergence is driven by
|
||||||
|
/// `--real`, not by the minting path.
|
||||||
|
#[test]
|
||||||
|
fn reproduce_real_sweep_family_re_derives_bit_identically() {
|
||||||
|
if !local_data_present() {
|
||||||
|
eprintln!("skip: no local data at {}", data_server::DEFAULT_DATA_PATH);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let (dir, _g) = fresh_project();
|
||||||
|
let fixture = format!("{}/examples/r_sma_open.json", env!("CARGO_MANIFEST_DIR"));
|
||||||
|
let mint = Command::new(BIN)
|
||||||
|
.args([
|
||||||
|
"sweep", &fixture,
|
||||||
|
"--real", "GER40",
|
||||||
|
"--from", GER40_SEPT2024_FROM_MS,
|
||||||
|
"--to", GER40_SEPT2024_TO_MS,
|
||||||
|
"--axis", "sma_signal.fast.length=2,4",
|
||||||
|
"--axis", "sma_signal.slow.length=8",
|
||||||
|
"--name", "repro_sweep",
|
||||||
|
])
|
||||||
|
.current_dir(dir)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
assert!(mint.status.success(), "sweep mint stderr: {}", String::from_utf8_lossy(&mint.stderr));
|
||||||
|
|
||||||
|
let id = family_id_of_kind(dir, "Sweep");
|
||||||
|
let out = Command::new(BIN).args(["reproduce", &id]).current_dir(dir).output().unwrap();
|
||||||
|
let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
|
||||||
|
assert!(
|
||||||
|
out.status.success(),
|
||||||
|
"reproduce of a real-data sweep family must succeed (exit 0), got {:?}\nstdout: {stdout}\nstderr: {}",
|
||||||
|
out.status.code(),
|
||||||
|
String::from_utf8_lossy(&out.stderr)
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
stdout.contains("reproduced 2/2 members bit-identically"),
|
||||||
|
"every member of a deterministic real-data sweep re-derives bit-identically: {stdout}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!stdout.contains("DIVERGED"),
|
||||||
|
"no member may DIVERGE — the run is deterministic, reproduce must use the recorded data source: {stdout}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Property: `aura reproduce` yields a handled outcome — a reproduction verdict
|
||||||
|
/// or a clean `aura:` refusal — for a real-data WalkForward family, and NEVER an
|
||||||
|
/// internal panic. A WalkForward member's OOS window is stored as real epoch-ns
|
||||||
|
/// bounds; reconstructing that window over the built-in synthetic walk (whose 60
|
||||||
|
/// bars fall entirely outside those bounds) yields an empty source, which today
|
||||||
|
/// reaches the unguarded `window_of(&s).expect("non-empty OOS window")` and
|
||||||
|
/// aborts the process (exit 101). Gated on the local GER40 archive
|
||||||
|
/// (skip-on-no-data convention); a ~135-day window is the minimum that stitches
|
||||||
|
/// at least one OOS window (IS 90d + OOS 30d).
|
||||||
|
#[test]
|
||||||
|
fn reproduce_real_walkforward_family_does_not_panic() {
|
||||||
|
if !local_data_present() {
|
||||||
|
eprintln!("skip: no local data at {}", data_server::DEFAULT_DATA_PATH);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// 2025-01-01 .. ~2025-05-16 (~135 days) — long enough for one IS(90d)+OOS(30d)
|
||||||
|
// walk-forward window over the local GER40 archive.
|
||||||
|
const WF_FROM_MS: &str = "1735689600000";
|
||||||
|
const WF_TO_MS: &str = "1747353600000";
|
||||||
|
let (dir, _g) = fresh_project();
|
||||||
|
let fixture = format!("{}/examples/r_sma_open.json", env!("CARGO_MANIFEST_DIR"));
|
||||||
|
let mint = Command::new(BIN)
|
||||||
|
.args([
|
||||||
|
"walkforward", &fixture,
|
||||||
|
"--real", "GER40",
|
||||||
|
"--from", WF_FROM_MS,
|
||||||
|
"--to", WF_TO_MS,
|
||||||
|
"--axis", "sma_signal.fast.length=2,4",
|
||||||
|
"--axis", "sma_signal.slow.length=8",
|
||||||
|
"--name", "repro_wf",
|
||||||
|
])
|
||||||
|
.current_dir(dir)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
assert!(mint.status.success(), "walkforward mint stderr: {}", String::from_utf8_lossy(&mint.stderr));
|
||||||
|
|
||||||
|
let id = family_id_of_kind(dir, "WalkForward");
|
||||||
|
let out = Command::new(BIN).args(["reproduce", &id]).current_dir(dir).output().unwrap();
|
||||||
|
let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
|
||||||
|
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
|
||||||
|
assert_ne!(
|
||||||
|
out.status.code(),
|
||||||
|
Some(101),
|
||||||
|
"reproduce of a WalkForward family must not panic (exit 101)\nstdout: {stdout}\nstderr: {stderr}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!stderr.contains("panicked") && !stderr.contains("non-empty OOS window"),
|
||||||
|
"no internal panic may leak from reproduce: {stderr}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
stdout.contains("reproduced") || stderr.starts_with("aura:"),
|
||||||
|
"reproduce must yield a verdict (stdout) or a clean `aura:` refusal (stderr): stdout={stdout} stderr={stderr}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// Property (#210 c0110 fieldtest, "the wrapped-name refusal mangles the axis
|
/// Property (#210 c0110 fieldtest, "the wrapped-name refusal mangles the axis
|
||||||
/// the user typed"): the dissolved real-data blueprint sweep types `--axis`
|
/// the user typed"): the dissolved real-data blueprint sweep types `--axis`
|
||||||
/// against the WRAPPED probe namespace (`sma_signal.fast.length`, what
|
/// against the WRAPPED probe namespace (`sma_signal.fast.length`, what
|
||||||
|
|||||||
Reference in New Issue
Block a user