diff --git a/Cargo.lock b/Cargo.lock index 44d3cec..ab4ebe9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,6 +138,7 @@ dependencies = [ "serde_json", "sha2", "toml", + "zip", ] [[package]] diff --git a/crates/aura-cli/Cargo.toml b/crates/aura-cli/Cargo.toml index d6d4d61..e988e30 100644 --- a/crates/aura-cli/Cargo.toml +++ b/crates/aura-cli/Cargo.toml @@ -53,3 +53,10 @@ toml = "0.8" # long-option abbreviation. Research-side CLI only (invariant 8): a dev-loop # compile tax, never a frozen-artifact tax. clap = { version = "4", features = ["derive"] } + +[dev-dependencies] +# zip: hand-packs a tiny synthetic M1 archive (tests/common/mod.rs) in the +# exact on-disk format data-server reads — already transitive via the +# data-server dependency above; declared directly here so test code may +# `use zip::...` (#250, no new dependency actually enters the build graph). +zip = "2" diff --git a/crates/aura-cli/tests/cli_run.rs b/crates/aura-cli/tests/cli_run.rs index f12342e..1fb67de 100644 --- a/crates/aura-cli/tests/cli_run.rs +++ b/crates/aura-cli/tests/cli_run.rs @@ -5,7 +5,7 @@ use std::path::Path; use std::process::Command; mod common; -use common::fresh_project; +use common::{fresh_project, fresh_project_with_data}; /// Path to the freshly-built `aura` binary (Cargo sets this env var for the test /// crate; the binary is named `aura` in `Cargo.toml`). @@ -3594,30 +3594,24 @@ fn generalize_distinct_invocations_persist_distinct_campaign_documents() { /// independent full window). A wrong intersection (or a panic on the /// per-symbol probe) in that fallback would only surface when `--from`/`--to` /// are absent — every other generalize e2e pins the explicit-window path, -/// leaving this one unexercised. Gated on local GER40/USDJPY data. +/// leaving this one unexercised. The property is the no-window RESOLUTION +/// mechanics, not any particular archive's span, so it runs against +/// `fresh_project_with_data`'s tiny synthetic two-symbol archive (#250) — +/// hostless, no data-refusal skip arm needed. #[test] fn generalize_without_explicit_window_resolves_a_shared_window_and_completes() { - let (cwd, _g) = fresh_project(); + let (cwd, _g) = fresh_project_with_data(); let fixture = format!("{}/examples/r_sma.json", env!("CARGO_MANIFEST_DIR")); let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) .args([ - "generalize", &fixture, "--real", "GER40,USDJPY", + "generalize", &fixture, "--real", "SYMA,SYMB", "--axis", "sma_signal.fast.length=3", "--axis", "sma_signal.slow.length=12", "--stop-length", "14", "--stop-k", "2.0", ]) .current_dir(&cwd) .output() .expect("spawn aura"); - if out.status.code() == Some(1) { - let stderr = String::from_utf8_lossy(&out.stderr); - assert!( - stderr.contains("no local data") || stderr.contains("no recorded geometry"), - "exit 1 must be a data refusal, got: {stderr}" - ); - eprintln!("skip: no local GER40/USDJPY data"); - return; - } - assert_eq!(out.status.code(), Some(0), "exit: {:?}", out.status); + assert_eq!(out.status.code(), Some(0), "exit: {:?}, stderr: {}", out.status, String::from_utf8_lossy(&out.stderr)); let stdout = String::from_utf8(out.stdout).expect("utf-8 stdout"); assert!(stdout.contains("\"generalize\":"), "aggregate object: {stdout}"); assert!(stdout.contains("\"n_instruments\":2"), "two instruments graded: {stdout}"); @@ -3633,24 +3627,21 @@ fn generalize_without_explicit_window_resolves_a_shared_window_and_completes() { /// instrument conflates the instrument axis with the period axis, so the common /// window is the only one that isolates what the verb measures. Today's fallback /// picks `symbols[0]`'s full window alone (the superseded shape test above): -/// listing the WIDER symbol first (`AAPL.US`, archive from 2006) keeps AAPL's -/// early start, whereas the intersection with the later-starting `GER40` (archive -/// from 2014) must start at GER40's first bar and end at whichever archive ends -/// first — so on the current tree the resolved window differs from the +/// listing the WIDER synthetic symbol first (`SYMA`, spanning 2024-01..08) keeps +/// SYMA's early start, whereas the intersection with `SYMB` (spanning 2024-03..06, +/// strictly inside SYMA's span) must start at SYMB's first bar and end at SYMB's +/// last — so on the current tree the resolved window differs from the /// intersection on both bounds. Each symbol's full window is resolved /// independently here: a single-symbol `sweep` with no window records it into its /// generated campaign document, in the same Unix-ms currency and via the same /// `campaign_window_ms(full_window)` probe generalize itself resolves against — /// so the expected intersection is derived from the live archive, never a -/// hard-coded span that shifts as the archive grows. Archive-gated; skips -/// cleanly on a data-less host. +/// hard-coded span. The archive's SPAN SHAPE is authored (#250, +/// `fresh_project_with_data`), not its size, so this runs hostless in +/// milliseconds instead of streaming two full host archives. #[test] fn generalize_without_explicit_window_resolves_the_intersection_of_all_symbols() { - if !local_data_present() { - eprintln!("skip: no local data at {}", data_server::DEFAULT_DATA_PATH); - return; - } - let (cwd, _g) = fresh_project(); + let (cwd, _g) = fresh_project_with_data(); let fixture = format!("{}/examples/r_sma.json", env!("CARGO_MANIFEST_DIR")); // The resolved shared window persists verbatim in the generated campaign @@ -3694,24 +3685,24 @@ fn generalize_without_explicit_window_resolves_the_intersection_of_all_symbols() window_of(name) }; - let aapl = probe("AAPL.US", "probe-aapl"); - let ger40 = probe("GER40", "probe-ger40"); + let syma = probe("SYMA", "probe-syma"); + let symb = probe("SYMB", "probe-symb"); // Precondition: the archives genuinely differ, so "the intersection" is a // distinct claim from "symbols[0]'s full window" (else the test is vacuous). assert_ne!( - aapl, ger40, - "AAPL.US {aapl:?} and GER40 {ger40:?} must span different windows for the test to bite" + syma, symb, + "SYMA {syma:?} and SYMB {symb:?} must span different windows for the test to bite" ); // The contract: the shared window is the intersection — latest start, earliest end. - let expected = (aapl.0.max(ger40.0), aapl.1.min(ger40.1)); + let expected = (syma.0.max(symb.0), syma.1.min(symb.1)); - // AAPL.US listed FIRST: today's `symbols[0]` fallback keeps AAPL's (wider) + // SYMA listed FIRST: today's `symbols[0]` fallback keeps SYMA's (wider) // window, which differs from the intersection on both bounds — so a pass here // can only mean the intersection semantics landed. let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) .args([ - "generalize", &fixture, "--real", "AAPL.US,GER40", + "generalize", &fixture, "--real", "SYMA,SYMB", "--axis", "sma_signal.fast.length=3", "--axis", "sma_signal.slow.length=12", "--name", "gen-both", ]) @@ -3727,8 +3718,8 @@ fn generalize_without_explicit_window_resolves_the_intersection_of_all_symbols() assert_eq!( resolved, expected, - "no-window generalize must resolve the INTERSECTION {expected:?} of AAPL.US {aapl:?} \ - and GER40 {ger40:?} — not symbols[0]'s ({aapl:?}) full window (resolved {resolved:?})" + "no-window generalize must resolve the INTERSECTION {expected:?} of SYMA {syma:?} \ + and SYMB {symb:?} — not symbols[0]'s ({syma:?}) full window (resolved {resolved:?})" ); } diff --git a/crates/aura-cli/tests/common/mod.rs b/crates/aura-cli/tests/common/mod.rs index 972eeec..7a7211b 100644 --- a/crates/aura-cli/tests/common/mod.rs +++ b/crates/aura-cli/tests/common/mod.rs @@ -52,6 +52,40 @@ impl Drop for FreshProject { } } +/// Mints a unique, empty tempdir under `std::env::temp_dir()`, named +/// `aura-e2e--` so concurrent test threads (same process) and +/// concurrent `cargo test` processes never collide. +fn mint_tempdir() -> PathBuf { + static COUNTER: AtomicUsize = AtomicUsize::new(0); + let root = std::env::temp_dir().join(format!( + "aura-e2e-{}-{}", + std::process::id(), + COUNTER.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&root).expect("create fresh project tempdir"); + root +} + +/// Writes the `Aura.toml` + `demo_signal.json` common to every fresh project: +/// an absolute `[nodes]` pointer at the shared, once-built fixture crate, and +/// — when `data_dir` is `Some` — a `[paths] data` override pointing at it +/// (relative, resolved against the invocation cwd, which every test that uses +/// this always sets to `root` itself via `.current_dir`). +fn write_project_files(root: &Path, data_dir: Option<&str>) { + let fixture = built_project(); + let data_line = data_dir.map(|d| format!("data = \"{d}\"\n")).unwrap_or_default(); + std::fs::write( + root.join("Aura.toml"), + format!( + "[paths]\nruns = \"runs\"\n{data_line}\n[nodes]\ncrates = [\"{}\"]\n", + fixture.display() + ), + ) + .expect("write fresh project Aura.toml"); + std::fs::copy(fixture.join("demo_signal.json"), root.join("demo_signal.json")) + .expect("copy demo_signal.json into the fresh project"); +} + /// Mint a unique tempdir holding a 2-line data-level project (`Aura.toml` /// with an absolute `[nodes]` pointer at the shared, once-built fixture /// crate) plus a copy of `demo_signal.json`. Every test gets its own `runs/` @@ -60,24 +94,26 @@ impl Drop for FreshProject { /// whole test body: `let (dir, _g) = fresh_project();`. #[allow(dead_code)] pub fn fresh_project() -> (PathBuf, FreshProject) { - static COUNTER: AtomicUsize = AtomicUsize::new(0); - let root = std::env::temp_dir().join(format!( - "aura-e2e-{}-{}", - std::process::id(), - COUNTER.fetch_add(1, Ordering::Relaxed) - )); - std::fs::create_dir_all(&root).expect("create fresh project tempdir"); - let fixture = built_project(); - std::fs::write( - root.join("Aura.toml"), - format!( - "[paths]\nruns = \"runs\"\n\n[nodes]\ncrates = [\"{}\"]\n", - fixture.display() - ), - ) - .expect("write fresh project Aura.toml"); - std::fs::copy(fixture.join("demo_signal.json"), root.join("demo_signal.json")) - .expect("copy demo_signal.json into the fresh project"); + let root = mint_tempdir(); + write_project_files(&root, None); + (root.clone(), FreshProject { root }) +} + +/// Like [`fresh_project`], but additionally generates a tiny synthetic M1 +/// archive under `/data/` (two symbols, `SYMA` spanning 2024-01..08 and +/// `SYMB` spanning 2024-03..06 — `SYMB` lies strictly inside `SYMA`) and +/// points `[paths] data` at it. For tests whose property is archive-window +/// (span) semantics, not archive size — no host tick-data mount needed, and +/// no `--real`-symbol data refusal is ever reachable, so gates on data +/// presence are dead code once a test switches to this helper (#250). +#[allow(dead_code)] +pub fn fresh_project_with_data() -> (PathBuf, FreshProject) { + let root = mint_tempdir(); + write_project_files(&root, Some("data")); + let data_dir = root.join("data"); + std::fs::create_dir_all(&data_dir).expect("create synthetic archive dir"); + synthetic_data::write_symbol_archive(&data_dir, "SYMA", (2024, 1), (2024, 8)); + synthetic_data::write_symbol_archive(&data_dir, "SYMB", (2024, 3), (2024, 6)); (root.clone(), FreshProject { root }) } @@ -108,3 +144,142 @@ impl Drop for ScratchGuard { } } } + +/// A tiny, deterministic synthetic M1 archive generator: hand-packs the exact +/// on-disk format `data-server` reads (its `loader.rs` and `records.rs`) — +/// one zip per `SYMBOL_YYYY_MM.m1` holding a single `.bin` entry of packed +/// 48-byte `RawM1Record`s, plus a `SYMBOL.meta.json` geometry sidecar — so +/// the two no-window `generalize` E2E tests get a real, tiny, hostless +/// archive instead of streaming the 6.6 GB host archive (#250). +#[allow(dead_code)] +mod synthetic_data { + use std::io::Write; + use std::path::Path; + + /// Days since the Unix epoch for a proleptic-Gregorian civil date. + /// Howard Hinnant's `days_from_civil` — dependency-free calendar math so + /// this module needs no `chrono` (data-server's own dependency, not + /// aura-cli's). + fn days_from_civil(y: i32, m: u32, d: u32) -> i64 { + let y = i64::from(if m <= 2 { y - 1 } else { y }); + let era = if y >= 0 { y } else { y - 399 } / 400; + let yoe = y - era * 400; // [0, 399] + let mp = (i64::from(m) + 9) % 12; // [0, 11], Mar=0 .. Feb=11 + let doy = (153 * mp + 2) / 5 + i64::from(d) - 1; // [0, 365] + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; // [0, 146096] + era * 146_097 + doe - 719_468 + } + + /// Unix milliseconds at `y-m-d hh:mm:00` UTC. + fn unix_ms(y: i32, m: u32, d: u32, hh: u32, mm: u32) -> i64 { + days_from_civil(y, m, d) * 86_400_000 + (i64::from(hh) * 3600 + i64::from(mm) * 60) * 1000 + } + + /// 0 = Sunday .. 6 = Saturday (1970-01-01 was a Thursday, day 0). + fn weekday(y: i32, m: u32, d: u32) -> i64 { + (days_from_civil(y, m, d) % 7 + 4 + 7) % 7 + } + + fn days_in_month(y: i32, m: u32) -> u32 { + match m { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + 2 if (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 => 29, + 2 => 28, + _ => unreachable!("month out of range: {m}"), + } + } + + /// A deterministic price at continuous minute offset `t` (minutes since + /// 2024-01-01T00:00 UTC, shared across both symbols so their overlapping + /// span carries the same underlying curve): a gentle upward trend plus a + /// 180-minute sine — short enough against a day's 480 tradeable minutes to + /// produce several SMA(3)/SMA(12) crossovers per day. + fn price_at(t_minutes: f64) -> f64 { + let hours = t_minutes / 60.0; + 100.0 + 0.01 * hours + 5.0 * (2.0 * std::f64::consts::PI * t_minutes / 180.0).sin() + } + + /// Packs one `RawM1Record`-shaped 48-byte little-endian record: `time` + /// (Delphi `TDateTime`, days since 1899-12-30 — the inverse of + /// `data_server::records::delphi_to_unix_ms`), `open/high/low/close`, + /// `spread` (f32), `volume` (i32). + fn pack_record(unix_ms_time: i64, open: f64, high: f64, low: f64, close: f64) -> [u8; 48] { + const DELPHI_EPOCH_OFFSET_DAYS: f64 = 25569.0; + const MS_PER_DAY: f64 = 86_400_000.0; + let dt = unix_ms_time as f64 / MS_PER_DAY + DELPHI_EPOCH_OFFSET_DAYS; + let mut rec = [0u8; 48]; + rec[0..8].copy_from_slice(&dt.to_le_bytes()); + rec[8..16].copy_from_slice(&open.to_le_bytes()); + rec[16..24].copy_from_slice(&high.to_le_bytes()); + rec[24..32].copy_from_slice(&low.to_le_bytes()); + rec[32..40].copy_from_slice(&close.to_le_bytes()); + rec[40..44].copy_from_slice(&0.5_f32.to_le_bytes()); + rec[44..48].copy_from_slice(&100_i32.to_le_bytes()); + rec + } + + /// One month's worth of weekday, 08:00-16:00 UTC, 1-minute bars (~10k). + fn month_records(year: i32, month: u32) -> Vec<[u8; 48]> { + let epoch_ms = unix_ms(2024, 1, 1, 0, 0); + let mut records = Vec::new(); + for day in 1..=days_in_month(year, month) { + if !(1..=5).contains(&weekday(year, month, day)) { + continue; // weekend + } + for hh in 8..16 { + for mm in 0..60 { + let ms = unix_ms(year, month, day, hh, mm); + let t = (ms - epoch_ms) as f64 / 60_000.0; + let (open, close) = (price_at(t), price_at(t + 1.0)); + let high = open.max(close) + 0.2; + let low = open.min(close) - 0.2; + records.push(pack_record(ms, open, high, low, close)); + } + } + } + records + } + + /// Writes one `SYMBOL_YYYY_MM.m1` zip (a single `.bin` entry of + /// packed records — mirrors `data-server`'s own `create_test_m1_zip` test + /// helper) under `data_dir`. + fn write_month_zip(data_dir: &Path, symbol: &str, year: i32, month: u32) { + let path = data_dir.join(format!("{symbol}_{year:04}_{month:02}.m1")); + let file = std::fs::File::create(&path).expect("create synthetic m1 zip"); + let mut zip = zip::ZipWriter::new(file); + zip.start_file::(format!("{symbol}.bin"), Default::default()) + .expect("start synthetic m1 zip entry"); + for rec in month_records(year, month) { + zip.write_all(&rec).expect("write synthetic m1 record"); + } + zip.finish().expect("finish synthetic m1 zip"); + } + + /// Writes every month in `[from, to]` (inclusive, `(year, month)` pairs) + /// for `symbol`, plus its `.meta.json` geometry sidecar — + /// GER40-shaped (digits 1, pip/tick/lot sized like an index CFD) so the + /// R/stop machinery never hits "no recorded geometry". + pub fn write_symbol_archive(data_dir: &Path, symbol: &str, from: (i32, u32), to: (i32, u32)) { + let (mut y, mut m) = from; + loop { + write_month_zip(data_dir, symbol, y, m); + if (y, m) == to { + break; + } + m += 1; + if m > 12 { + m = 1; + y += 1; + } + } + std::fs::write( + data_dir.join(format!("{symbol}.meta.json")), + format!( + "{{\"schemaVersion\":2,\"symbol\":\"{symbol}\",\"digits\":1,\"pipSize\":1,\ + \"tickSize\":0.1,\"lotSize\":1,\"baseAsset\":\"{symbol}\",\"quoteAsset\":\"EUR\"}}" + ), + ) + .expect("write synthetic geometry sidecar"); + } +}