fix(bench): make the cli_fixed_cost fingerprint invariant across build commits
Caught by the harness itself on its first post-commit run: the aura run record line embeds manifest.commit — the binary's compile-time build sha (crates/aura-cli/build.rs, the invariant-8 audit trail) — so hashing the raw line flipped the fingerprint on every rebuild at a new commit and reported a phantom correctness change (exit 1) at each cycle close. The rebase onto the #191 main exposed it; nothing in #191 touched the record. RED-first: run_line_fingerprint_is_invariant_across_build_commits pins the symptom (two lines differing only in manifest.commit must hash identically); the fix extracts run_line_fingerprint, which parses the line, blanks that one volatile field, re-serializes (serde_json maps are sorted, so the bytes are deterministic), and hashes. A missing manifest.commit is a hard error, so a future record-schema move fails loudly instead of silently passing. The cli_fixed_cost baseline is re-pinned under the new definition — ratify: the fingerprint DEFINITION changed (provenance excluded, results still covered, pinned by run_line_fingerprint_still_detects_result_changes); no measured behaviour of aura moved. refs #251
This commit is contained in:
@@ -1,16 +1,16 @@
|
||||
{
|
||||
"surface": "cli_fixed_cost",
|
||||
"metrics": {
|
||||
"help_ms": 1.53201,
|
||||
"run_ms": 3.37904
|
||||
"help_ms": 1.5094509999999999,
|
||||
"run_ms": 3.679743
|
||||
},
|
||||
"fingerprint": "run_line_fnv=ef791827583f616b",
|
||||
"fingerprint": "run_line_fnv=792dc4434f8e3dee",
|
||||
"reps": 3,
|
||||
"host": {
|
||||
"hostname": "Raki",
|
||||
"nproc": 24
|
||||
},
|
||||
"profile": "release",
|
||||
"commit": "d62dede",
|
||||
"commit": "13d8500",
|
||||
"date": "2026-07-17"
|
||||
}
|
||||
|
||||
@@ -11,6 +11,25 @@ use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
/// Fingerprint of an `aura run` record line, invariant across rebuilds: the
|
||||
/// line's `manifest.commit` is the aura binary's compile-time build sha
|
||||
/// (crates/aura-cli/build.rs — the invariant-8 audit trail), so hashing the
|
||||
/// raw line would flip the fingerprint on EVERY new commit and report a
|
||||
/// phantom correctness change at each cycle close. Parse, blank that one
|
||||
/// volatile field, re-serialize (serde_json's map is sorted — deterministic
|
||||
/// bytes), then hash.
|
||||
pub fn run_line_fingerprint(line: &str) -> Result<String, String> {
|
||||
let mut v: serde_json::Value =
|
||||
serde_json::from_str(line).map_err(|e| format!("run record line parses: {e}"))?;
|
||||
let commit = v
|
||||
.get_mut("manifest")
|
||||
.and_then(|m| m.get_mut("commit"))
|
||||
.ok_or("run record line carries manifest.commit")?;
|
||||
*commit = serde_json::Value::String(String::new());
|
||||
let canonical = serde_json::to_string(&v).expect("re-serializing a parsed Value cannot fail");
|
||||
Ok(format!("run_line_fnv={:016x}", fnv1a(canonical.as_bytes())))
|
||||
}
|
||||
|
||||
/// Spawn samples folded into one repetition: single-digit-millisecond spawns
|
||||
/// jitter well past the 10% NOTICE threshold, so each repetition reports the
|
||||
/// median over a batch instead of one throw of the scheduler dice.
|
||||
@@ -60,5 +79,44 @@ pub fn rep(bin: &Path) -> Result<RepOutcome, String> {
|
||||
let mut metrics = BTreeMap::new();
|
||||
metrics.insert("help_ms".to_string(), median(help_walls));
|
||||
metrics.insert("run_ms".to_string(), median(run_walls));
|
||||
Ok(RepOutcome { metrics, fingerprint: format!("run_line_fnv={:016x}", fnv1a(line.as_bytes())) })
|
||||
Ok(RepOutcome { metrics, fingerprint: run_line_fingerprint(&line)? })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The symptom this fixes, pinned: two record lines that differ ONLY in
|
||||
/// `manifest.commit` (a rebuild at a new commit) must fingerprint
|
||||
/// identically — the raw-line hash flipped here and reported a phantom
|
||||
/// correctness change on the first post-commit bench run.
|
||||
#[test]
|
||||
fn run_line_fingerprint_is_invariant_across_build_commits() {
|
||||
let at_a = r#"{"manifest":{"commit":"aaaa","seed":0},"metrics":{"total_pips":0.5}}"#;
|
||||
let at_b = r#"{"manifest":{"commit":"bbbb","seed":0},"metrics":{"total_pips":0.5}}"#;
|
||||
assert_eq!(
|
||||
run_line_fingerprint(at_a).expect("parses"),
|
||||
run_line_fingerprint(at_b).expect("parses"),
|
||||
);
|
||||
}
|
||||
|
||||
/// The fingerprint still catches what it exists for: a computed-result
|
||||
/// change (any non-provenance byte) yields a different hash.
|
||||
#[test]
|
||||
fn run_line_fingerprint_still_detects_result_changes() {
|
||||
let base = r#"{"manifest":{"commit":"aaaa","seed":0},"metrics":{"total_pips":0.5}}"#;
|
||||
let drifted = r#"{"manifest":{"commit":"aaaa","seed":0},"metrics":{"total_pips":0.6}}"#;
|
||||
assert_ne!(
|
||||
run_line_fingerprint(base).expect("parses"),
|
||||
run_line_fingerprint(drifted).expect("parses"),
|
||||
);
|
||||
}
|
||||
|
||||
/// A line without the provenance field is malformed input, not a silent
|
||||
/// pass — the guard that keeps the blanking honest if the record schema
|
||||
/// ever moves.
|
||||
#[test]
|
||||
fn run_line_fingerprint_rejects_a_line_without_manifest_commit() {
|
||||
assert!(run_line_fingerprint(r#"{"metrics":{"total_pips":0.5}}"#).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user