From 13d85004022c520c029f180c17f74b1af2729e90 Mon Sep 17 00:00:00 2001 From: claude Date: Fri, 17 Jul 2026 15:48:04 +0200 Subject: [PATCH] =?UTF-8?q?audit:=20cycle-close=20tidy=20for=20#251=20?= =?UTF-8?q?=E2=80=94=20fingerprint=20honesty,=20lockstep=20+=20surface-lis?= =?UTF-8?q?t=20guards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Architect drift review (cycle bfb8648..0389399): no domain-invariant breach. What holds: aura-bench is bin-only dev tooling with zero production reverse deps (invariant 8/C13); every committed surface runs on seeded synthetic data in scratch temp projects (invariant 9); run_reps enforces cross-rep fingerprint identity and the campaign fingerprint carries integer ordinals across the child boundary, consistent with C1 and its cross-command ULP carve-out; zip/clap per-case dependency notes in place (C16), and the ledger correctly gains no entry — tooling lives in the project facts + README. Three drift items, resolved in this commit: - [medium] the winner_fingerprint doc claimed walk-forward regressions fail the bench; the walk-forward stage in fact emits an empty realization record, so coverage is only indirect (via the bootstrap trade count). Doc now states the gap honestly; the engine-side fix is parked on #279. - [low] the fingerprint's lenient JSON parse would silently erode on a registry-schema rename: a new lockstep test builds a real CampaignRunRecord from the aura-registry/aura-analysis/aura-engine types (new dev-deps) and asserts the extracted fingerprint, so a rename breaks this crate's tests instead. - [low] LIBRARY_SURFACES was hand-synced with measure_surface: a new test pins that every listed surface measures without a binary path. Regression gates on the closing tree: cargo test --workspace 1379 green, clippy -D warnings clean, doc build clean, aura-bench run exit 0 with five 'fingerprint OK' at <=2% drift. No baseline moved in this commit — the bench baselines were pinned and verified in the feature commit. refs #251 --- Cargo.lock | 2 + crates/aura-bench/Cargo.toml | 9 ++ crates/aura-bench/src/main.rs | 16 ++++ crates/aura-bench/src/surfaces/campaign.rs | 99 +++++++++++++++++++++- 4 files changed, 122 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19b0306..eeb7a7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,9 +109,11 @@ dependencies = [ name = "aura-bench" version = "0.1.0" dependencies = [ + "aura-analysis", "aura-core", "aura-engine", "aura-ingest", + "aura-registry", "aura-std", "clap", "serde", diff --git a/crates/aura-bench/Cargo.toml b/crates/aura-bench/Cargo.toml index daba124..0de1979 100644 --- a/crates/aura-bench/Cargo.toml +++ b/crates/aura-bench/Cargo.toml @@ -27,3 +27,12 @@ clap = { version = "4", features = ["derive"] } # under the C16 per-case policy: aura-bench is a dev tool outside every deploy # path, and zip is already in the build graph via data-server. zip = "2" + +[dev-dependencies] +# Lockstep guard for the campaign fingerprint's stdout-schema coupling: the +# winner_fingerprint test constructs a REAL CampaignRunRecord (aura-registry) +# with a real FamilySelection (aura-analysis) and serializes it, so a field +# rename in those crates fails this crate's tests instead of silently eroding +# fingerprint coverage. +aura-registry = { path = "../aura-registry" } +aura-analysis = { path = "../aura-analysis" } diff --git a/crates/aura-bench/src/main.rs b/crates/aura-bench/src/main.rs index 7f06ab5..3dedf19 100644 --- a/crates/aura-bench/src/main.rs +++ b/crates/aura-bench/src/main.rs @@ -392,6 +392,22 @@ mod tests { /// the point of the in-process surfaces. Proven by using a nonexistent /// `ws_root` — `resolve_aura_bin` would error immediately if `prepare` /// ever called it for this filter. + /// Pins the hand-synced `LIBRARY_SURFACES` list against `measure_surface` + /// itself: every listed name must be a registered surface AND actually + /// measurable without a real binary path (quick mode, 1 rep). A surface + /// added to the list without being binary-free fails here — the silent + /// failure mode would otherwise be a needless release build on every + /// library-only invocation. + #[test] + fn library_surfaces_measure_without_the_binary() { + for name in LIBRARY_SURFACES { + assert!(SURFACES.contains(name), "{name} must be a registered surface"); + let fake_bin = Path::new("/nonexistent-aura-binary-for-library-surface-test"); + measure_surface(name, Sizing { quick: true }, 1, fake_bin) + .unwrap_or_else(|e| panic!("library surface {name} must run without the binary: {e}")); + } + } + #[test] fn prepare_skips_binary_resolution_for_library_only_surfaces() { let fake_root = Path::new("/nonexistent-ws-root-for-aura-bench-test"); diff --git a/crates/aura-bench/src/surfaces/campaign.rs b/crates/aura-bench/src/surfaces/campaign.rs index 35a429b..2b90e7c 100644 --- a/crates/aura-bench/src/surfaces/campaign.rs +++ b/crates/aura-bench/src/surfaces/campaign.rs @@ -211,10 +211,14 @@ fn build_scratch(bin: &Path, sizing: Sizing, process_doc: &str) -> Result<(Scrat /// Extract the determinism fingerprint from the always-on final /// `{"campaign_run":...}` stdout line: per stage the selection's winner /// ordinal (`w`), the survivor-ordinal list (`v[..]`), and the bootstrap's -/// trade count (`b`, pooled or summed across per-survivor entries). Folding -/// all three realization layers in means a wrong-result regression in the -/// sweep, gate, walk-forward, or bootstrap machinery fails the bench — not -/// just a wrong sweep winner. +/// trade count (`b`, pooled or summed across per-survivor entries). A +/// wrong-result regression in the sweep, gate, or bootstrap machinery +/// therefore fails the bench — not just a wrong sweep winner. The +/// walk-forward stage currently emits an empty realization record +/// (no selection, no survivors, no bootstrap), so it contributes nothing +/// directly and is covered only indirectly, through the bootstrap trade +/// count that depends on its out-of-sample output (coverage gap parked on +/// the tracker, #279). pub fn winner_fingerprint(stdout: &str) -> Result { let line = stdout .lines() @@ -355,6 +359,93 @@ mod tests { } } + /// Lockstep guard for the stdout-schema coupling: build a REAL + /// `CampaignRunRecord` out of the aura-registry / aura-analysis / + /// aura-engine types the CLI serializes, wrap it the way the record line + /// is printed, and assert `winner_fingerprint` extracts every layer. A + /// field rename in any of those crates now breaks this test (compile or + /// assert) instead of silently eroding fingerprint coverage through the + /// lenient JSON parse. + #[test] + fn winner_fingerprint_stays_in_lockstep_with_the_registry_record_types() { + use aura_analysis::{FamilySelection, SelectionMode}; + use aura_engine::{MetricStats, RBootstrap}; + use aura_registry::{CampaignRunRecord, CellRealization, StageRealization, StageSelection}; + + let stats = MetricStats { mean: 0.1, p5: 0.0, p25: 0.05, p50: 0.1, p75: 0.15, p95: 0.2 }; + let record = CampaignRunRecord { + campaign: "camp-id".into(), + process: "proc-id".into(), + run: 0, + seed: 7, + cells: vec![CellRealization { + strategy: "bp-id".into(), + instrument: "BSYMA".into(), + window_ms: (1, 2), + stages: vec![ + StageRealization { + block: "std::sweep".into(), + family_id: None, + survivor_ordinals: None, + selection: Some(StageSelection { + winner_ordinal: 4, + params: vec![], + selection: FamilySelection { + selection_metric: "sqn_normalized".into(), + n_trials: 25, + raw_winner_metric: 0.5, + mode: SelectionMode::Argmax, + deflated_score: None, + overfit_probability: None, + n_resamples: None, + block_len: None, + seed: None, + neighbourhood_score: None, + n_neighbours: None, + }, + }), + bootstrap: None, + window_faults: vec![], + }, + StageRealization { + block: "std::gate".into(), + family_id: None, + survivor_ordinals: Some(vec![0, 2]), + selection: None, + bootstrap: None, + window_faults: vec![], + }, + StageRealization { + block: "std::monte_carlo".into(), + family_id: None, + survivor_ordinals: None, + selection: None, + bootstrap: Some(aura_registry::StageBootstrap::PooledOos(RBootstrap { + e_r: stats, + prob_le_zero: 0.3, + n_trades: 42, + block_len: 5, + n_resamples: 1000, + })), + window_faults: vec![], + }, + ], + regime: None, + regime_ordinal: 0, + fault: None, + coverage: None, + }], + generalizations: vec![], + trace_name: None, + }; + let line = format!( + "{{\"campaign_run\":{}}}", + serde_json::to_string(&record).expect("record serializes") + ); + let fp = winner_fingerprint(&line).expect("the real record shape must parse"); + assert_eq!(fp, "cells=1 c0s0w4 c0s1v[0.2] c0s2b42"); + } + #[test] fn process_docs_are_valid_json() { for doc in [SWEEP_PROCESS_DOC, HEAVY_PROCESS_DOC, HEAVY_PROCESS_DOC_QUICK] {