fix(bench): make the cli_fixed_cost fingerprint invariant across build commits

Caught by the harness itself on its first post-commit run: the aura run
record line embeds manifest.commit — the binary's compile-time build sha
(crates/aura-cli/build.rs, the invariant-8 audit trail) — so hashing the raw
line flipped the fingerprint on every rebuild at a new commit and reported a
phantom correctness change (exit 1) at each cycle close. The rebase onto the
#191 main exposed it; nothing in #191 touched the record.

RED-first: run_line_fingerprint_is_invariant_across_build_commits pins the
symptom (two lines differing only in manifest.commit must hash identically);
the fix extracts run_line_fingerprint, which parses the line, blanks that
one volatile field, re-serializes (serde_json maps are sorted, so the bytes
are deterministic), and hashes. A missing manifest.commit is a hard error,
so a future record-schema move fails loudly instead of silently passing.
The cli_fixed_cost baseline is re-pinned under the new definition — ratify:
the fingerprint DEFINITION changed (provenance excluded, results still
covered, pinned by run_line_fingerprint_still_detects_result_changes); no
measured behaviour of aura moved.

refs #251
This commit is contained in:
2026-07-17 18:16:56 +02:00
parent 13d8500402
commit 16ca4e103e
2 changed files with 63 additions and 5 deletions
@@ -1,16 +1,16 @@
{
"surface": "cli_fixed_cost",
"metrics": {
"help_ms": 1.53201,
"run_ms": 3.37904
"help_ms": 1.5094509999999999,
"run_ms": 3.679743
},
"fingerprint": "run_line_fnv=ef791827583f616b",
"fingerprint": "run_line_fnv=792dc4434f8e3dee",
"reps": 3,
"host": {
"hostname": "Raki",
"nproc": 24
},
"profile": "release",
"commit": "d62dede",
"commit": "13d8500",
"date": "2026-07-17"
}
+59 -1
View File
@@ -11,6 +11,25 @@ use std::collections::BTreeMap;
use std::path::Path;
use std::time::Instant;
/// Fingerprint of an `aura run` record line, invariant across rebuilds: the
/// line's `manifest.commit` is the aura binary's compile-time build sha
/// (crates/aura-cli/build.rs — the invariant-8 audit trail), so hashing the
/// raw line would flip the fingerprint on EVERY new commit and report a
/// phantom correctness change at each cycle close. Parse, blank that one
/// volatile field, re-serialize (serde_json's map is sorted — deterministic
/// bytes), then hash.
pub fn run_line_fingerprint(line: &str) -> Result<String, String> {
let mut v: serde_json::Value =
serde_json::from_str(line).map_err(|e| format!("run record line parses: {e}"))?;
let commit = v
.get_mut("manifest")
.and_then(|m| m.get_mut("commit"))
.ok_or("run record line carries manifest.commit")?;
*commit = serde_json::Value::String(String::new());
let canonical = serde_json::to_string(&v).expect("re-serializing a parsed Value cannot fail");
Ok(format!("run_line_fnv={:016x}", fnv1a(canonical.as_bytes())))
}
/// Spawn samples folded into one repetition: single-digit-millisecond spawns
/// jitter well past the 10% NOTICE threshold, so each repetition reports the
/// median over a batch instead of one throw of the scheduler dice.
@@ -60,5 +79,44 @@ pub fn rep(bin: &Path) -> Result<RepOutcome, String> {
let mut metrics = BTreeMap::new();
metrics.insert("help_ms".to_string(), median(help_walls));
metrics.insert("run_ms".to_string(), median(run_walls));
Ok(RepOutcome { metrics, fingerprint: format!("run_line_fnv={:016x}", fnv1a(line.as_bytes())) })
Ok(RepOutcome { metrics, fingerprint: run_line_fingerprint(&line)? })
}
#[cfg(test)]
mod tests {
use super::*;
/// The symptom this fixes, pinned: two record lines that differ ONLY in
/// `manifest.commit` (a rebuild at a new commit) must fingerprint
/// identically — the raw-line hash flipped here and reported a phantom
/// correctness change on the first post-commit bench run.
#[test]
fn run_line_fingerprint_is_invariant_across_build_commits() {
let at_a = r#"{"manifest":{"commit":"aaaa","seed":0},"metrics":{"total_pips":0.5}}"#;
let at_b = r#"{"manifest":{"commit":"bbbb","seed":0},"metrics":{"total_pips":0.5}}"#;
assert_eq!(
run_line_fingerprint(at_a).expect("parses"),
run_line_fingerprint(at_b).expect("parses"),
);
}
/// The fingerprint still catches what it exists for: a computed-result
/// change (any non-provenance byte) yields a different hash.
#[test]
fn run_line_fingerprint_still_detects_result_changes() {
let base = r#"{"manifest":{"commit":"aaaa","seed":0},"metrics":{"total_pips":0.5}}"#;
let drifted = r#"{"manifest":{"commit":"aaaa","seed":0},"metrics":{"total_pips":0.6}}"#;
assert_ne!(
run_line_fingerprint(base).expect("parses"),
run_line_fingerprint(drifted).expect("parses"),
);
}
/// A line without the provenance field is malformed input, not a silent
/// pass — the guard that keeps the blanking honest if the record schema
/// ever moves.
#[test]
fn run_line_fingerprint_rejects_a_line_without_manifest_commit() {
assert!(run_line_fingerprint(r#"{"metrics":{"total_pips":0.5}}"#).is_err());
}
}