Brummel/Aura
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases Wiki Activity

test(cli): synthetic per-test M1 archive — no-window generalize tests go hostless

Browse Source 
The two generalize E2E tests whose property is no-window span
resolution (shared-window fallback, intersection semantics) no longer
stream the 6.6 GB host archive: fresh_project_with_data() generates a
deterministic two-symbol M1 archive (SYMA 2024-01..08, SYMB 2024-03..06
strictly inside it, so the intersection differs from symbols[0]'s
window on both bounds) in the exact zip/48-byte-record format
data-server reads, plus geometry sidecars, under the per-test project's
paths.data. Both tests drop their local_data_present gates — they now
run on any host, data mount or not.

~35s and ~50s become 0.74s and 1.14s; the full cli_run binary lands at
~25s (211s at the branch base). Full workspace suite: 247s -> 45.7s.

refs #250
This commit is contained in:
Claude
2026-07-13 15:24:00 +02:00
parent 25b8354959
commit c31e946bb7
4 changed files with 226 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+1
View File
@@ -138,6 +138,7 @@ dependencies = [
"serde_json",
"sha2",
"toml",
"zip",
]
[[package]]
crates/aura-cli/Cargo.toml
+7
View File
@@ -53,3 +53,10 @@ toml = "0.8"
# long-option abbreviation. Research-side CLI only (invariant 8): a dev-loop
# compile tax, never a frozen-artifact tax.
clap = { version = "4", features = ["derive"] }
[dev-dependencies]
# zip: hand-packs a tiny synthetic M1 archive (tests/common/mod.rs) in the
# exact on-disk format data-server reads — already transitive via the
# data-server dependency above; declared directly here so test code may
# `use zip::...` (#250, no new dependency actually enters the build graph).
zip = "2"
crates/aura-cli/tests/cli_run.rs
+25 -34
View File
@@ -5,7 +5,7 @@ use std::path::Path;
use std::process::Command;
mod common;
use common::fresh_project;
use common::{fresh_project, fresh_project_with_data};
/// Path to the freshly-built `aura` binary (Cargo sets this env var for the test
/// crate; the binary is named `aura` in `Cargo.toml`).
@@ -3594,30 +3594,24 @@ fn generalize_distinct_invocations_persist_distinct_campaign_documents() {
/// independent full window). A wrong intersection (or a panic on the
/// per-symbol probe) in that fallback would only surface when `--from`/`--to`
/// are absent — every other generalize e2e pins the explicit-window path,
/// leaving this one unexercised. Gated on local GER40/USDJPY data.
/// leaving this one unexercised. The property is the no-window RESOLUTION
/// mechanics, not any particular archive's span, so it runs against
/// `fresh_project_with_data`'s tiny synthetic two-symbol archive (#250) —
/// hostless, no data-refusal skip arm needed.
#[test]
fn generalize_without_explicit_window_resolves_a_shared_window_and_completes() {
let (cwd, _g) = fresh_project();
let (cwd, _g) = fresh_project_with_data();
let fixture = format!("{}/examples/r_sma.json", env!("CARGO_MANIFEST_DIR"));
let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura"))
.args([
"generalize", &fixture, "--real", "GER40,USDJPY",
"generalize", &fixture, "--real", "SYMA,SYMB",
"--axis", "sma_signal.fast.length=3", "--axis", "sma_signal.slow.length=12",
"--stop-length", "14", "--stop-k", "2.0",
])
.current_dir(&cwd)
.output()
.expect("spawn aura");
if out.status.code() == Some(1) {
let stderr = String::from_utf8_lossy(&out.stderr);
assert!(
stderr.contains("no local data") || stderr.contains("no recorded geometry"),
"exit 1 must be a data refusal, got: {stderr}"
);
eprintln!("skip: no local GER40/USDJPY data");
return;
}
assert_eq!(out.status.code(), Some(0), "exit: {:?}", out.status);
assert_eq!(out.status.code(), Some(0), "exit: {:?}, stderr: {}", out.status, String::from_utf8_lossy(&out.stderr));
let stdout = String::from_utf8(out.stdout).expect("utf-8 stdout");
assert!(stdout.contains("\"generalize\":"), "aggregate object: {stdout}");
assert!(stdout.contains("\"n_instruments\":2"), "two instruments graded: {stdout}");
@@ -3633,24 +3627,21 @@ fn generalize_without_explicit_window_resolves_a_shared_window_and_completes() {
/// instrument conflates the instrument axis with the period axis, so the common
/// window is the only one that isolates what the verb measures. Today's fallback
/// picks `symbols[0]`'s full window alone (the superseded shape test above):
/// listing the WIDER symbol first (`AAPL.US`, archive from 2006) keeps AAPL's
/// early start, whereas the intersection with the later-starting `GER40` (archive
/// from 2014) must start at GER40's first bar and end at whichever archive ends
/// first — so on the current tree the resolved window differs from the
/// listing the WIDER synthetic symbol first (`SYMA`, spanning 2024-01..08) keeps
/// SYMA's early start, whereas the intersection with `SYMB` (spanning 2024-03..06,
/// strictly inside SYMA's span) must start at SYMB's first bar and end at SYMB's
/// last — so on the current tree the resolved window differs from the
/// intersection on both bounds. Each symbol's full window is resolved
/// independently here: a single-symbol `sweep` with no window records it into its
/// generated campaign document, in the same Unix-ms currency and via the same
/// `campaign_window_ms(full_window)` probe generalize itself resolves against —
/// so the expected intersection is derived from the live archive, never a
/// hard-coded span that shifts as the archive grows. Archive-gated; skips
/// cleanly on a data-less host.
/// hard-coded span. The archive's SPAN SHAPE is authored (#250,
/// `fresh_project_with_data`), not its size, so this runs hostless in
/// milliseconds instead of streaming two full host archives.
#[test]
fn generalize_without_explicit_window_resolves_the_intersection_of_all_symbols() {
if !local_data_present() {
eprintln!("skip: no local data at {}", data_server::DEFAULT_DATA_PATH);
return;
}
let (cwd, _g) = fresh_project();
let (cwd, _g) = fresh_project_with_data();
let fixture = format!("{}/examples/r_sma.json", env!("CARGO_MANIFEST_DIR"));
// The resolved shared window persists verbatim in the generated campaign
@@ -3694,24 +3685,24 @@ fn generalize_without_explicit_window_resolves_the_intersection_of_all_symbols()
window_of(name)
};
let aapl = probe("AAPL.US", "probe-aapl");
let ger40 = probe("GER40", "probe-ger40");
let syma = probe("SYMA", "probe-syma");
let symb = probe("SYMB", "probe-symb");
// Precondition: the archives genuinely differ, so "the intersection" is a
// distinct claim from "symbols[0]'s full window" (else the test is vacuous).
assert_ne!(
aapl, ger40,
"AAPL.US {aapl:?} and GER40 {ger40:?} must span different windows for the test to bite"
syma, symb,
"SYMA {syma:?} and SYMB {symb:?} must span different windows for the test to bite"
);
// The contract: the shared window is the intersection — latest start, earliest end.
let expected = (aapl.0.max(ger40.0), aapl.1.min(ger40.1));
let expected = (syma.0.max(symb.0), syma.1.min(symb.1));
// AAPL.US listed FIRST: today's `symbols[0]` fallback keeps AAPL's (wider)
// SYMA listed FIRST: today's `symbols[0]` fallback keeps SYMA's (wider)
// window, which differs from the intersection on both bounds — so a pass here
// can only mean the intersection semantics landed.
let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura"))
.args([
"generalize", &fixture, "--real", "AAPL.US,GER40",
"generalize", &fixture, "--real", "SYMA,SYMB",
"--axis", "sma_signal.fast.length=3", "--axis", "sma_signal.slow.length=12",
"--name", "gen-both",
])
@@ -3727,8 +3718,8 @@ fn generalize_without_explicit_window_resolves_the_intersection_of_all_symbols()
assert_eq!(
resolved, expected,
"no-window generalize must resolve the INTERSECTION {expected:?} of AAPL.US {aapl:?} \
and GER40 {ger40:?} — not symbols[0]'s ({aapl:?}) full window (resolved {resolved:?})"
"no-window generalize must resolve the INTERSECTION {expected:?} of SYMA {syma:?} \
and SYMB {symb:?} — not symbols[0]'s ({syma:?}) full window (resolved {resolved:?})"
);
}
crates/aura-cli/tests/common/mod.rs
+193 -18
View File
@@ -52,6 +52,40 @@ impl Drop for FreshProject {
}
}
/// Mints a unique, empty tempdir under `std::env::temp_dir()`, named
/// `aura-e2e-<pid>-<counter>` so concurrent test threads (same process) and
/// concurrent `cargo test` processes never collide.
fn mint_tempdir() -> PathBuf {
static COUNTER: AtomicUsize = AtomicUsize::new(0);
let root = std::env::temp_dir().join(format!(
"aura-e2e-{}-{}",
std::process::id(),
COUNTER.fetch_add(1, Ordering::Relaxed)
));
std::fs::create_dir_all(&root).expect("create fresh project tempdir");
root
}
/// Writes the `Aura.toml` + `demo_signal.json` common to every fresh project:
/// an absolute `[nodes]` pointer at the shared, once-built fixture crate, and
/// — when `data_dir` is `Some` — a `[paths] data` override pointing at it
/// (relative, resolved against the invocation cwd, which every test that uses
/// this always sets to `root` itself via `.current_dir`).
fn write_project_files(root: &Path, data_dir: Option<&str>) {
let fixture = built_project();
let data_line = data_dir.map(|d| format!("data = \"{d}\"\n")).unwrap_or_default();
std::fs::write(
root.join("Aura.toml"),
format!(
"[paths]\nruns = \"runs\"\n{data_line}\n[nodes]\ncrates = [\"{}\"]\n",
fixture.display()
),
)
.expect("write fresh project Aura.toml");
std::fs::copy(fixture.join("demo_signal.json"), root.join("demo_signal.json"))
.expect("copy demo_signal.json into the fresh project");
}
/// Mint a unique tempdir holding a 2-line data-level project (`Aura.toml`
/// with an absolute `[nodes]` pointer at the shared, once-built fixture
/// crate) plus a copy of `demo_signal.json`. Every test gets its own `runs/`
@@ -60,24 +94,26 @@ impl Drop for FreshProject {
/// whole test body: `let (dir, _g) = fresh_project();`.
#[allow(dead_code)]
pub fn fresh_project() -> (PathBuf, FreshProject) {
static COUNTER: AtomicUsize = AtomicUsize::new(0);
let root = std::env::temp_dir().join(format!(
"aura-e2e-{}-{}",
std::process::id(),
COUNTER.fetch_add(1, Ordering::Relaxed)
));
std::fs::create_dir_all(&root).expect("create fresh project tempdir");
let fixture = built_project();
std::fs::write(
root.join("Aura.toml"),
format!(
"[paths]\nruns = \"runs\"\n\n[nodes]\ncrates = [\"{}\"]\n",
fixture.display()
),
)
.expect("write fresh project Aura.toml");
std::fs::copy(fixture.join("demo_signal.json"), root.join("demo_signal.json"))
.expect("copy demo_signal.json into the fresh project");
let root = mint_tempdir();
write_project_files(&root, None);
(root.clone(), FreshProject { root })
}
/// Like [`fresh_project`], but additionally generates a tiny synthetic M1
/// archive under `<root>/data/` (two symbols, `SYMA` spanning 2024-01..08 and
/// `SYMB` spanning 2024-03..06 — `SYMB` lies strictly inside `SYMA`) and
/// points `[paths] data` at it. For tests whose property is archive-window
/// (span) semantics, not archive size — no host tick-data mount needed, and
/// no `--real`-symbol data refusal is ever reachable, so gates on data
/// presence are dead code once a test switches to this helper (#250).
#[allow(dead_code)]
pub fn fresh_project_with_data() -> (PathBuf, FreshProject) {
let root = mint_tempdir();
write_project_files(&root, Some("data"));
let data_dir = root.join("data");
std::fs::create_dir_all(&data_dir).expect("create synthetic archive dir");
synthetic_data::write_symbol_archive(&data_dir, "SYMA", (2024, 1), (2024, 8));
synthetic_data::write_symbol_archive(&data_dir, "SYMB", (2024, 3), (2024, 6));
(root.clone(), FreshProject { root })
}
@@ -108,3 +144,142 @@ impl Drop for ScratchGuard {
}
}
}
/// A tiny, deterministic synthetic M1 archive generator: hand-packs the exact
/// on-disk format `data-server` reads (its `loader.rs` and `records.rs`) —
/// one zip per `SYMBOL_YYYY_MM.m1` holding a single `.bin` entry of packed
/// 48-byte `RawM1Record`s, plus a `SYMBOL.meta.json` geometry sidecar — so
/// the two no-window `generalize` E2E tests get a real, tiny, hostless
/// archive instead of streaming the 6.6 GB host archive (#250).
#[allow(dead_code)]
mod synthetic_data {
use std::io::Write;
use std::path::Path;
/// Days since the Unix epoch for a proleptic-Gregorian civil date.
/// Howard Hinnant's `days_from_civil` — dependency-free calendar math so
/// this module needs no `chrono` (data-server's own dependency, not
/// aura-cli's).
fn days_from_civil(y: i32, m: u32, d: u32) -> i64 {
let y = i64::from(if m <= 2 { y - 1 } else { y });
let era = if y >= 0 { y } else { y - 399 } / 400;
let yoe = y - era * 400; // [0, 399]
let mp = (i64::from(m) + 9) % 12; // [0, 11], Mar=0 .. Feb=11
let doy = (153 * mp + 2) / 5 + i64::from(d) - 1; // [0, 365]
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; // [0, 146096]
era * 146_097 + doe - 719_468
}
/// Unix milliseconds at `y-m-d hh:mm:00` UTC.
fn unix_ms(y: i32, m: u32, d: u32, hh: u32, mm: u32) -> i64 {
days_from_civil(y, m, d) * 86_400_000 + (i64::from(hh) * 3600 + i64::from(mm) * 60) * 1000
}
/// 0 = Sunday .. 6 = Saturday (1970-01-01 was a Thursday, day 0).
fn weekday(y: i32, m: u32, d: u32) -> i64 {
(days_from_civil(y, m, d) % 7 + 4 + 7) % 7
}
fn days_in_month(y: i32, m: u32) -> u32 {
match m {
1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
4 | 6 | 9 | 11 => 30,
2 if (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 => 29,
2 => 28,
_ => unreachable!("month out of range: {m}"),
}
}
/// A deterministic price at continuous minute offset `t` (minutes since
/// 2024-01-01T00:00 UTC, shared across both symbols so their overlapping
/// span carries the same underlying curve): a gentle upward trend plus a
/// 180-minute sine — short enough against a day's 480 tradeable minutes to
/// produce several SMA(3)/SMA(12) crossovers per day.
fn price_at(t_minutes: f64) -> f64 {
let hours = t_minutes / 60.0;
100.0 + 0.01 * hours + 5.0 * (2.0 * std::f64::consts::PI * t_minutes / 180.0).sin()
}
/// Packs one `RawM1Record`-shaped 48-byte little-endian record: `time`
/// (Delphi `TDateTime`, days since 1899-12-30 — the inverse of
/// `data_server::records::delphi_to_unix_ms`), `open/high/low/close`,
/// `spread` (f32), `volume` (i32).
fn pack_record(unix_ms_time: i64, open: f64, high: f64, low: f64, close: f64) -> [u8; 48] {
const DELPHI_EPOCH_OFFSET_DAYS: f64 = 25569.0;
const MS_PER_DAY: f64 = 86_400_000.0;
let dt = unix_ms_time as f64 / MS_PER_DAY + DELPHI_EPOCH_OFFSET_DAYS;
let mut rec = [0u8; 48];
rec[0..8].copy_from_slice(&dt.to_le_bytes());
rec[8..16].copy_from_slice(&open.to_le_bytes());
rec[16..24].copy_from_slice(&high.to_le_bytes());
rec[24..32].copy_from_slice(&low.to_le_bytes());
rec[32..40].copy_from_slice(&close.to_le_bytes());
rec[40..44].copy_from_slice(&0.5_f32.to_le_bytes());
rec[44..48].copy_from_slice(&100_i32.to_le_bytes());
rec
}
/// One month's worth of weekday, 08:00-16:00 UTC, 1-minute bars (~10k).
fn month_records(year: i32, month: u32) -> Vec<[u8; 48]> {
let epoch_ms = unix_ms(2024, 1, 1, 0, 0);
let mut records = Vec::new();
for day in 1..=days_in_month(year, month) {
if !(1..=5).contains(&weekday(year, month, day)) {
continue; // weekend
}
for hh in 8..16 {
for mm in 0..60 {
let ms = unix_ms(year, month, day, hh, mm);
let t = (ms - epoch_ms) as f64 / 60_000.0;
let (open, close) = (price_at(t), price_at(t + 1.0));
let high = open.max(close) + 0.2;
let low = open.min(close) - 0.2;
records.push(pack_record(ms, open, high, low, close));
}
}
}
records
}
/// Writes one `SYMBOL_YYYY_MM.m1` zip (a single `<SYMBOL>.bin` entry of
/// packed records — mirrors `data-server`'s own `create_test_m1_zip` test
/// helper) under `data_dir`.
fn write_month_zip(data_dir: &Path, symbol: &str, year: i32, month: u32) {
let path = data_dir.join(format!("{symbol}_{year:04}_{month:02}.m1"));
let file = std::fs::File::create(&path).expect("create synthetic m1 zip");
let mut zip = zip::ZipWriter::new(file);
zip.start_file::<String, ()>(format!("{symbol}.bin"), Default::default())
.expect("start synthetic m1 zip entry");
for rec in month_records(year, month) {
zip.write_all(&rec).expect("write synthetic m1 record");
}
zip.finish().expect("finish synthetic m1 zip");
}
/// Writes every month in `[from, to]` (inclusive, `(year, month)` pairs)
/// for `symbol`, plus its `<SYMBOL>.meta.json` geometry sidecar —
/// GER40-shaped (digits 1, pip/tick/lot sized like an index CFD) so the
/// R/stop machinery never hits "no recorded geometry".
pub fn write_symbol_archive(data_dir: &Path, symbol: &str, from: (i32, u32), to: (i32, u32)) {
let (mut y, mut m) = from;
loop {
write_month_zip(data_dir, symbol, y, m);
if (y, m) == to {
break;
}
m += 1;
if m > 12 {
m = 1;
y += 1;
}
}
std::fs::write(
data_dir.join(format!("{symbol}.meta.json")),
format!(
"{{\"schemaVersion\":2,\"symbol\":\"{symbol}\",\"digits\":1,\"pipSize\":1,\
\"tickSize\":0.1,\"lotSize\":1,\"baseAsset\":\"{symbol}\",\"quoteAsset\":\"EUR\"}}"
),
)
.expect("write synthetic geometry sidecar");
}
}