diff --git a/Cargo.lock b/Cargo.lock index 40fcb3f..19b0306 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -105,6 +105,20 @@ dependencies = [ "serde_json", ] +[[package]] +name = "aura-bench" +version = "0.1.0" +dependencies = [ + "aura-core", + "aura-engine", + "aura-ingest", + "aura-std", + "clap", + "serde", + "serde_json", + "zip", +] + [[package]] name = "aura-campaign" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index b606106..99302f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "crates/aura-registry", "crates/aura-research", "crates/aura-campaign", + "crates/aura-bench", ] [workspace.package] diff --git a/crates/aura-bench/Cargo.toml b/crates/aura-bench/Cargo.toml new file mode 100644 index 0000000..daba124 --- /dev/null +++ b/crates/aura-bench/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "aura-bench" +edition.workspace = true +version.workspace = true +license.workspace = true +publish.workspace = true + +[[bin]] +name = "aura-bench" +path = "src/main.rs" + +# Dev-only measurement tool (issue #251): benches the engine / ingest / campaign +# surfaces against committed baselines under baselines/. No production crate +# depends on this crate; nothing here reaches a frozen deploy artifact (C13). +[dependencies] +aura-core = { path = "../aura-core" } +aura-engine = { path = "../aura-engine" } +aura-std = { path = "../aura-std" } +aura-ingest = { path = "../aura-ingest" } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +# clap: same CLI surface convention as aura-cli (usage errors exit 2). +clap = { version = "4", features = ["derive"] } +# zip: writes the synthetic 48-byte M1 archives the ingest/campaign surfaces +# measure — the same on-disk format the test suites hand-pack as a dev-dep +# (aura-cli/aura-ingest). First runtime (non-dev) use in the workspace, admitted +# under the C16 per-case policy: aura-bench is a dev tool outside every deploy +# path, and zip is already in the build graph via data-server. +zip = "2" diff --git a/crates/aura-bench/src/baseline.rs b/crates/aura-bench/src/baseline.rs new file mode 100644 index 0000000..04af3eb --- /dev/null +++ b/crates/aura-bench/src/baseline.rs @@ -0,0 +1,171 @@ +//! Baseline documents and the report-only comparison core. Pure functions — +//! everything here is unit-tested without running a single benchmark. +//! +//! The full CLI glue that wires this module's public surface into the +//! surfaces' measurement loop lands in Task 7; until then nothing outside +//! `#[cfg(test)]` calls into it, so the module is exempted from the +//! dead-code lint here rather than left to trip the workspace's +//! `-D warnings` clippy gate. +#![allow(dead_code)] + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// One committed baseline: `baselines/.json`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct BaselineDoc { + pub surface: String, + /// Flat metric map, e.g. `bars_per_s`, `wall_s`, `peak_rss_mb`. + pub metrics: BTreeMap, + /// Opaque determinism fingerprint; compared for byte equality. + pub fingerprint: String, + pub reps: u32, + pub host: HostInfo, + pub profile: String, + pub commit: String, + pub date: String, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct HostInfo { + pub hostname: String, + pub nproc: u32, +} + +/// One measurement result, produced by a surface run. +#[derive(Clone, Debug, PartialEq)] +pub struct Measured { + pub surface: String, + pub metrics: BTreeMap, + pub fingerprint: String, +} + +/// Timing drift at or beyond this fraction is reported as a NOTICE line. +pub const NOTICE_THRESHOLD: f64 = 0.10; + +#[derive(Clone, Debug, PartialEq)] +pub struct Compared { + /// Per-metric relative delta, `(name, (measured - baseline) / baseline)`. + /// Metrics missing on either side are skipped. + pub deltas: Vec<(String, f64)>, + /// Metric names whose |delta| >= NOTICE_THRESHOLD. + pub notices: Vec, + pub fingerprint_changed: bool, + pub host_mismatch: bool, +} + +pub fn compare(baseline: &BaselineDoc, measured: &Measured, host: &HostInfo) -> Compared { + let mut deltas = Vec::new(); + let mut notices = Vec::new(); + for (name, base) in &baseline.metrics { + let Some(now) = measured.metrics.get(name) else { continue }; + if *base == 0.0 { + continue; // no meaningful relative delta + } + let d = (now - base) / base; + if d.abs() >= NOTICE_THRESHOLD { + notices.push(name.clone()); + } + deltas.push((name.clone(), d)); + } + Compared { + deltas, + notices, + fingerprint_changed: baseline.fingerprint != measured.fingerprint, + host_mismatch: baseline.host != *host, + } +} + +/// The process exit code for a full run: 1 if ANY surface's fingerprint +/// changed, else 0. (Infra errors exit 2 before this is ever consulted; +/// timing drift is report-only by design — the #251 cadence decision.) +pub fn exit_code(compared: &[Option]) -> i32 { + if compared.iter().flatten().any(|c| c.fingerprint_changed) { 1 } else { 0 } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn doc(metrics: &[(&str, f64)], fp: &str) -> BaselineDoc { + BaselineDoc { + surface: "s".into(), + metrics: metrics.iter().map(|(k, v)| (k.to_string(), *v)).collect(), + fingerprint: fp.into(), + reps: 3, + host: HostInfo { hostname: "h".into(), nproc: 24 }, + profile: "release".into(), + commit: "abc".into(), + date: "2026-07-17".into(), + } + } + + fn measured(metrics: &[(&str, f64)], fp: &str) -> Measured { + Measured { + surface: "s".into(), + metrics: metrics.iter().map(|(k, v)| (k.to_string(), *v)).collect(), + fingerprint: fp.into(), + } + } + + fn host() -> HostInfo { + HostInfo { hostname: "h".into(), nproc: 24 } + } + + #[test] + fn compare_reports_relative_drift_per_metric() { + let c = compare(&doc(&[("wall_s", 10.0)], "f"), &measured(&[("wall_s", 10.5)], "f"), &host()); + assert_eq!(c.deltas.len(), 1); + assert_eq!(c.deltas[0].0, "wall_s"); + assert!((c.deltas[0].1 - 0.05).abs() < 1e-12, "5% drift, got {}", c.deltas[0].1); + assert!(c.notices.is_empty(), "5% is below the 10% notice bar"); + assert!(!c.fingerprint_changed); + assert!(!c.host_mismatch); + } + + #[test] + fn compare_flags_notice_at_ten_percent() { + let c = compare(&doc(&[("wall_s", 10.0)], "f"), &measured(&[("wall_s", 11.0)], "f"), &host()); + assert_eq!(c.notices, vec!["wall_s".to_string()]); + } + + #[test] + fn compare_detects_fingerprint_change() { + let c = compare(&doc(&[], "old"), &measured(&[], "new"), &host()); + assert!(c.fingerprint_changed); + } + + #[test] + fn compare_flags_host_mismatch_informationally() { + let other = HostInfo { hostname: "elsewhere".into(), nproc: 8 }; + let c = compare(&doc(&[], "f"), &measured(&[], "f"), &other); + assert!(c.host_mismatch); + assert!(!c.fingerprint_changed); + } + + #[test] + fn compare_skips_metrics_missing_on_either_side() { + let c = compare( + &doc(&[("only_base", 1.0)], "f"), + &measured(&[("only_measured", 2.0)], "f"), + &host(), + ); + assert!(c.deltas.is_empty()); + } + + #[test] + fn exit_code_is_one_only_on_fingerprint_change() { + let clean = compare(&doc(&[], "f"), &measured(&[], "f"), &host()); + let changed = compare(&doc(&[], "f"), &measured(&[], "g"), &host()); + assert_eq!(exit_code(&[Some(clean.clone()), None]), 0); + assert_eq!(exit_code(&[Some(clean), Some(changed)]), 1); + } + + #[test] + fn baseline_doc_round_trips_through_json() { + let d = doc(&[("bars_per_s", 812345.0)], "last=0.1 n=5"); + let json = serde_json::to_string_pretty(&d).expect("serialize"); + let back: BaselineDoc = serde_json::from_str(&json).expect("parse"); + assert_eq!(d, back); + } +} diff --git a/crates/aura-bench/src/main.rs b/crates/aura-bench/src/main.rs new file mode 100644 index 0000000..58d7c24 --- /dev/null +++ b/crates/aura-bench/src/main.rs @@ -0,0 +1,17 @@ +//! `aura-bench` — the wall-clock benchmark harness with committed baselines +//! (issue #251, phase 1). Report-only: timing drift never fails; a determinism +//! fingerprint mismatch does (exit 1), so a fast-but-wrong binary fails the +//! bench instead of pinning a bogus baseline. + +mod baseline; +mod synth; + +fn main() { + // Full CLI glue lands with the surfaces (Task 7). The skeleton only + // guards the profile: wall-clock numbers from a debug build measure the + // profile, not the code. + if cfg!(debug_assertions) { + eprintln!("aura-bench measures nothing in a debug build; run via `cargo run --release -p aura-bench`"); + std::process::exit(2); + } +} diff --git a/crates/aura-bench/src/synth.rs b/crates/aura-bench/src/synth.rs new file mode 100644 index 0000000..d2acc71 --- /dev/null +++ b/crates/aura-bench/src/synth.rs @@ -0,0 +1,256 @@ +//! Deterministic synthetic inputs: the seeded price walk (engine surface), the +//! 48-byte M1 zip archive writer (ingest + campaign surfaces), and the scratch +//! temp-dir guard. Everything here is a pure function of its arguments — same +//! bytes on every host, every run (C1 is what makes the engine benchmarkable). +//! +//! The CLI glue that wires this module's public surface into the surfaces' +//! measurement loop lands in Task 7; until then nothing outside `#[cfg(test)]` +//! calls into it, so the module is exempted from the dead-code lint here +//! rather than left to trip the workspace's `-D warnings` clippy gate. +#![allow(dead_code)] + +use aura_core::{Scalar, Timestamp}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +/// A seeded LCG random walk as a `VecSource` stream: 1-minute spacing in +/// epoch-ns, prices around 100. Same constants, same bytes, everywhere. +pub fn synthetic_walk(n: usize) -> Vec<(Timestamp, Scalar)> { + let mut x: u64 = 0x9E37_79B9_7F4A_7C15; + let mut price = 100.0_f64; + (0..n) + .map(|i| { + x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + let step = ((x >> 11) as f64 / (1u64 << 53) as f64) - 0.5; + price += step; + (Timestamp(i as i64 * 60_000_000_000), Scalar::f64(price)) + }) + .collect() +} + +/// FNV-1a over raw bytes — the fingerprint hash for child-process stdout. +pub fn fnv1a(bytes: &[u8]) -> u64 { + let mut h = 0xcbf2_9ce4_8422_2325_u64; + for b in bytes { + h ^= u64::from(*b); + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h +} + +/// A self-deleting scratch directory under the system temp dir. +pub struct ScratchDir(pub PathBuf); + +impl ScratchDir { + pub fn new(tag: &str) -> std::io::Result { + let base = std::env::temp_dir().join(format!( + "aura-bench-{}-{}-{}", + std::process::id(), + tag, + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + std::fs::create_dir_all(&base)?; + Ok(Self(base)) + } +} + +impl Drop for ScratchDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// Days since the Unix epoch for a proleptic-Gregorian civil date +/// (Howard Hinnant's `days_from_civil` — dependency-free calendar math). +fn days_from_civil(y: i32, m: u32, d: u32) -> i64 { + let y = i64::from(if m <= 2 { y - 1 } else { y }); + let era = if y >= 0 { y } else { y - 399 } / 400; + let yoe = y - era * 400; // [0, 399] + let mp = (i64::from(m) + 9) % 12; // [0, 11], Mar=0 .. Feb=11 + let doy = (153 * mp + 2) / 5 + i64::from(d) - 1; // [0, 365] + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; // [0, 146096] + era * 146_097 + doe - 719_468 +} + +/// Unix milliseconds at `y-m-d hh:mm:00` UTC. +pub fn unix_ms(y: i32, m: u32, d: u32, hh: u32, mm: u32) -> i64 { + days_from_civil(y, m, d) * 86_400_000 + (i64::from(hh) * 3600 + i64::from(mm) * 60) * 1000 +} + +/// 0 = Sunday .. 6 = Saturday (1970-01-01 was a Thursday, day 0). +fn weekday(y: i32, m: u32, d: u32) -> i64 { + (days_from_civil(y, m, d) % 7 + 4 + 7) % 7 +} + +fn days_in_month(y: i32, m: u32) -> u32 { + match m { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + 2 if (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 => 29, + 2 => 28, + _ => unreachable!("month out of range: {m}"), + } +} + +/// A deterministic price at continuous minute offset `t` (minutes since +/// 2024-01-01T00:00 UTC): a gentle upward trend plus a 180-minute sine. +fn price_at(t_minutes: f64) -> f64 { + let hours = t_minutes / 60.0; + 100.0 + 0.01 * hours + 5.0 * (2.0 * std::f64::consts::PI * t_minutes / 180.0).sin() +} + +/// Packs one `RawM1Record`-shaped 48-byte little-endian record: `time` +/// (Delphi `TDateTime`, days since 1899-12-30), `open/high/low/close`, +/// `spread` (f32), `volume` (i32). +fn pack_record(unix_ms_time: i64, open: f64, high: f64, low: f64, close: f64) -> [u8; 48] { + const DELPHI_EPOCH_OFFSET_DAYS: f64 = 25569.0; + const MS_PER_DAY: f64 = 86_400_000.0; + let dt = unix_ms_time as f64 / MS_PER_DAY + DELPHI_EPOCH_OFFSET_DAYS; + let mut rec = [0u8; 48]; + rec[0..8].copy_from_slice(&dt.to_le_bytes()); + rec[8..16].copy_from_slice(&open.to_le_bytes()); + rec[16..24].copy_from_slice(&high.to_le_bytes()); + rec[24..32].copy_from_slice(&low.to_le_bytes()); + rec[32..40].copy_from_slice(&close.to_le_bytes()); + rec[40..44].copy_from_slice(&0.5_f32.to_le_bytes()); + rec[44..48].copy_from_slice(&100_i32.to_le_bytes()); + rec +} + +/// One month's worth of weekday, 08:00-16:00 UTC, 1-minute bars (~10k). +fn month_records(year: i32, month: u32) -> Vec<[u8; 48]> { + let epoch_ms = unix_ms(2024, 1, 1, 0, 0); + let mut records = Vec::new(); + for day in 1..=days_in_month(year, month) { + if !(1..=5).contains(&weekday(year, month, day)) { + continue; // weekend + } + for hh in 8..16 { + for mm in 0..60 { + let ms = unix_ms(year, month, day, hh, mm); + let t = (ms - epoch_ms) as f64 / 60_000.0; + let (open, close) = (price_at(t), price_at(t + 1.0)); + let high = open.max(close) + 0.2; + let low = open.min(close) - 0.2; + records.push(pack_record(ms, open, high, low, close)); + } + } + } + records +} + +/// Writes one `SYMBOL_YYYY_MM.m1` zip (a single `.bin` entry). +fn write_month_zip(data_dir: &Path, symbol: &str, year: i32, month: u32) { + let path = data_dir.join(format!("{symbol}_{year:04}_{month:02}.m1")); + let file = std::fs::File::create(&path).expect("create synthetic m1 zip"); + let mut zip = zip::ZipWriter::new(file); + zip.start_file::(format!("{symbol}.bin"), Default::default()) + .expect("start synthetic m1 zip entry"); + for rec in month_records(year, month) { + zip.write_all(&rec).expect("write synthetic m1 record"); + } + zip.finish().expect("finish synthetic m1 zip"); +} + +/// Writes every month in `[from, to]` (inclusive `(year, month)` pairs) for +/// `symbol`, plus its `.meta.json` geometry sidecar (GER40-shaped). +pub fn write_symbol_archive(data_dir: &Path, symbol: &str, from: (i32, u32), to: (i32, u32)) { + let (mut y, mut m) = from; + loop { + write_month_zip(data_dir, symbol, y, m); + if (y, m) == to { + break; + } + m += 1; + if m > 12 { + m = 1; + y += 1; + } + } + std::fs::write( + data_dir.join(format!("{symbol}.meta.json")), + format!( + "{{\"schemaVersion\":2,\"symbol\":\"{symbol}\",\"digits\":1,\"pipSize\":1,\ + \"tickSize\":0.1,\"lotSize\":1,\"baseAsset\":\"{symbol}\",\"quoteAsset\":\"EUR\"}}" + ), + ) + .expect("write synthetic geometry sidecar"); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn synthetic_walk_is_deterministic_and_sized() { + let a = synthetic_walk(1000); + let b = synthetic_walk(1000); + assert_eq!(a.len(), 1000); + assert_eq!(a, b, "same seed, same bytes"); + assert_eq!(a[0].0, Timestamp(0)); + assert_eq!(a[1].0, Timestamp(60_000_000_000)); + } + + #[test] + fn write_symbol_archive_names_months_and_sidecar() { + let dir = ScratchDir::new("synthtest").expect("scratch dir"); + write_symbol_archive(&dir.0, "TSYM", (2024, 11), (2025, 2)); + let mut names: Vec = std::fs::read_dir(&dir.0) + .expect("read scratch") + .map(|e| e.expect("entry").file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + assert_eq!( + names, + vec![ + "TSYM.meta.json".to_string(), + "TSYM_2024_11.m1".to_string(), + "TSYM_2024_12.m1".to_string(), + "TSYM_2025_01.m1".to_string(), + "TSYM_2025_02.m1".to_string(), + ], + "month rollover + sidecar" + ); + } + + #[test] + fn fnv1a_is_stable() { + assert_eq!(fnv1a(b""), 0xcbf2_9ce4_8422_2325); + assert_eq!(fnv1a(b"a"), fnv1a(b"a")); + assert_ne!(fnv1a(b"a"), fnv1a(b"b")); + } + + /// #251: the synthetic archive is the ingest surface's benchmark input + /// (Task 7), so its bytes must be more than internally self-consistent — + /// they must actually parse through the real production M1 reader + /// (`aura_ingest::DataServer`/`load_m1_window`), the same path a real + /// GER40 archive goes through. This is not a round trip against our own + /// `pack_record`: it is the production parser recovering `pack_record`'s + /// 48-byte layout (Delphi `TDateTime`, 4 little-endian f64 prices) and + /// the `price_at` OHLC shape (`low <= open,close <= high` on every bar) — + /// pins the record layout against future drift in either writer or + /// reader. + #[test] + fn synthetic_archive_parses_through_the_real_ingest_reader() { + let dir = ScratchDir::new("ingest-roundtrip").expect("scratch dir"); + write_symbol_archive(&dir.0, "BENCHSYM", (2024, 1), (2024, 1)); + let server = std::sync::Arc::new(aura_ingest::DataServer::new(&dir.0)); + let cols = aura_ingest::load_m1_window(&server, "BENCHSYM", None, None) + .expect("the synthetic archive is a recognized data source"); + + assert!(!cols.close.is_empty(), "January 2024 has weekday 08:00-16:00 bars"); + for i in 0..cols.close.len() { + assert!( + cols.high[i] >= cols.open[i] && cols.high[i] >= cols.close[i], + "bar {i}: high must dominate open/close" + ); + assert!( + cols.low[i] <= cols.open[i] && cols.low[i] <= cols.close[i], + "bar {i}: low must be dominated by open/close" + ); + } + } +} diff --git a/crates/aura-bench/tests/profile_guard.rs b/crates/aura-bench/tests/profile_guard.rs new file mode 100644 index 0000000..8b8e37e --- /dev/null +++ b/crates/aura-bench/tests/profile_guard.rs @@ -0,0 +1,28 @@ +//! Integration test: drive the built `aura-bench` binary as a downstream +//! script would (spawned as a fresh process, no in-process shortcuts). + +use std::process::Command; + +/// Path to the freshly-built `aura-bench` binary (Cargo sets this env var +/// for the test crate; the binary target is named `aura-bench`). +const BIN: &str = env!("CARGO_BIN_EXE_aura-bench"); + +/// #251: a debug-build wall-clock number measures the build profile, not the +/// code, so the harness must refuse to run at all rather than silently +/// producing a number that could later be trusted (or committed) as a +/// baseline. This pins the profile guard's observable contract — exit code 2 +/// (the usage-error convention aura-bench shares with `aura-cli`) and a +/// stderr message naming both the cause ("debug build") and the fix +/// ("--release") — so a future CLI-wiring change (the surfaces land in +/// Task 7) cannot silently drop or weaken the guard. +#[test] +fn refuses_to_run_in_a_debug_build() { + // `cargo test` builds test-harness binaries (and their bin-target + // dependencies) in the dev profile by default, so this binary always + // has debug_assertions on — deterministic across hosts and CI. + let out = Command::new(BIN).output().expect("spawn aura-bench"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!(out.status.code(), Some(2), "stderr: {stderr}"); + assert!(stderr.contains("debug build"), "stderr: {stderr}"); + assert!(stderr.contains("--release"), "stderr: {stderr}"); +}