diff --git a/crates/aura-cli/src/main.rs b/crates/aura-cli/src/main.rs index a22ace8..f605119 100644 --- a/crates/aura-cli/src/main.rs +++ b/crates/aura-cli/src/main.rs @@ -5550,6 +5550,16 @@ mod tests { ); } + /// #158 acc 1 (cross-surface agreement): `topology_hash` (from a live `Composite`, the + /// run/sweep path) and the op-script `graph introspect --content-id` surface are the + /// SAME hash of the SAME canonical bytes — both go through the one `content_id` + /// primitive. Pins that the two command paths cannot silently drift apart. + #[test] + fn topology_hash_is_the_content_id_of_the_canonical_form() { + let sig = stage1_signal(Some(2), Some(4)); + assert_eq!(topology_hash(&sig), content_id(&blueprint_to_json(&sig).expect("serializes"))); + } + #[test] fn parse_mc_args_bare_and_name_route_to_synthetic() { assert_eq!(parse_mc_args(&[]), Ok(McArgs::Synthetic { name: "mc".to_string(), persist: false })); diff --git a/crates/aura-cli/tests/cli_run.rs b/crates/aura-cli/tests/cli_run.rs index 078e1f7..ca66f68 100644 --- a/crates/aura-cli/tests/cli_run.rs +++ b/crates/aura-cli/tests/cli_run.rs @@ -3506,3 +3506,66 @@ fn aura_reproduce_rejects_an_unknown_family() { ); let _ = std::fs::remove_dir_all(&cwd); } + +/// E2E (refuse-don't-guess): `aura reproduce` on a HARD-WIRED sweep family — its members +/// carry no `topology_hash` (not built from a blueprint) — exits non-zero, naming that the +/// member is not a generated run. Content-addressed reproduction covers only generated runs. +#[test] +fn aura_reproduce_rejects_a_non_generated_family() { + let cwd = temp_cwd("reproduce_non_generated"); + let sweep = std::process::Command::new(BIN) + .args(["sweep", "--name", "hw"]) + .current_dir(&cwd) + .output() + .expect("run aura sweep"); + assert!(sweep.status.success(), "hard-wired sweep ok: {}", String::from_utf8_lossy(&sweep.stderr)); + let out = std::process::Command::new(BIN) + .args(["reproduce", "hw-0"]) + .current_dir(&cwd) + .output() + .expect("run aura reproduce"); + assert!(!out.status.success(), "a non-generated family exits non-zero"); + assert!( + String::from_utf8_lossy(&out.stderr).contains("not a generated run"), + "names the cause: {}", + String::from_utf8_lossy(&out.stderr) + ); + let _ = std::fs::remove_dir_all(&cwd); +} + +/// E2E (refuse-don't-guess): `aura reproduce` when the content-addressed blueprint was +/// removed from the store exits non-zero, naming the missing blueprint — never re-runs +/// against a guessed topology (C10/C18). +#[test] +fn aura_reproduce_rejects_a_missing_stored_blueprint() { + let cwd = temp_cwd("reproduce_missing_store"); + let fixture = format!("{}/tests/fixtures/stage1_signal_open.json", env!("CARGO_MANIFEST_DIR")); + let sweep = std::process::Command::new(BIN) + .args([ + "sweep", + &fixture, + "--axis", + "stage1_signal.fast.length=2", + "--axis", + "stage1_signal.slow.length=4", + "--name", + "bp", + ]) + .current_dir(&cwd) + .output() + .expect("run aura sweep"); + assert!(sweep.status.success(), "blueprint sweep ok: {}", String::from_utf8_lossy(&sweep.stderr)); + std::fs::remove_dir_all(cwd.join("runs").join("blueprints")).expect("remove the blueprint store"); + let out = std::process::Command::new(BIN) + .args(["reproduce", "bp-0"]) + .current_dir(&cwd) + .output() + .expect("run aura reproduce"); + assert!(!out.status.success(), "a missing stored blueprint exits non-zero"); + assert!( + String::from_utf8_lossy(&out.stderr).contains("missing from store"), + "names the cause: {}", + String::from_utf8_lossy(&out.stderr) + ); + let _ = std::fs::remove_dir_all(&cwd); +} diff --git a/docs/design/INDEX.md b/docs/design/INDEX.md index fd52b52..2dcb5f0 100644 --- a/docs/design/INDEX.md +++ b/docs/design/INDEX.md @@ -1258,7 +1258,8 @@ byte-identical — C14/C23). The cross-instrument *generalization score* (worst- floor + sign-agreement + per-instrument breakdown) is a **recomputable aggregate** over those members, not a persisted family-level record — distinct from #144/#145's per-winner selection annotation on `RunManifest.selection`. -Deferred (Non-goals): content-addressed identity + replay-dedup; the "run-diff" +Deferred (Non-goals): replay-dedup (content-addressed *identity* shipped — #158, +cycle 0094, Realization below); the "run-diff" depth and ranking families against each other (cross-family, vs. within-family); and a live producer for the flat `runs.jsonl` standalone-run path — no CLI command writes it (sweep/walkforward persist to the family store; `aura run` does not @@ -1288,6 +1289,32 @@ topologies (C21/C24), `commit` no longer identifies the graph, so the manifest m topology's identity. The format and the carrier are C24's design; recorded here as the reproduction requirement it must meet. +**Realization (2026-07-01, cycle 0094 — content-addressed reproduction of a generated +run, #158).** A blueprint sweep's topology is now **content-addressed**: the canonical +`blueprint_to_json` bytes are stored once, keyed by the `topology_hash` the manifest +already carries, in a **dumb bytes-by-key store** beside the run registry +(`runs/blueprints/.json` — `Registry::put_blueprint`/`get_blueprint`, aura-registry; +no `sha2`, no parse — the caller owns the hash, and reproduction's bit-identical compare is +the integrity check). `aura reproduce ` re-derives every persisted member: load +the member's blueprint by its `topology_hash`, reconstruct the sweep point from the +recorded params, re-run through the **same** `run_blueprint_member` the live sweep uses +(so bit-identity is by construction, C1), and compare metrics — refuse-don't-guess on an +unknown id / missing stored blueprint (exit 2), DIVERGED → exit 1. The manifest + the +content-addressed store + the commit are the complete re-derivation recipe (C18). One +blueprint is stored per family (all members share the signal `topology_hash` — C11/C12 +dedup). `aura graph introspect --content-id` exposes the same id for an op-script, via the +one `content_id` primitive `topology_hash` also uses (acc 1); a Tier-1 optional the +blueprint does not use leaves the id byte-stable (acc 3, composing #156/#164). +`serde_json/float_roundtrip` is enabled so stored f64 metrics round-trip exactly through +`families.jsonl` — the precondition for a bit-identical compare (C1). **Signal-only this +cycle**: the id covers the signal blueprint; the fixed Stage-1-R scaffolding stays +commit-identified (it is not yet blueprint-data, C24). Whole-harness / structural-axis +content-addressing, and whether the id should exclude the composite **debug-name** (a +non-load-bearing symbol, invariant 11 — an op-script and the Rust builder produce +different canonical forms today), remain deferred (#171, #170). Reproduction is proven on +synthetic (deterministic) data; recorded-dataset reproduction rides the DataServer seam +(#124). + ### C19 — Bootstrap: blueprint → instance (recursive) **Guarantee.** Construction is a distinct phase, recursive at every level. Each node type has a **factory** `params → sized concrete node` (e.g. `SMA(length)` @@ -1863,10 +1890,12 @@ consumes the graph). Monte-Carlo / walk-forward over a loaded blueprint are the per verb (a tracked follow-on); per-member trace-writing on `--trace` and an axis-name discovery surface are tracked debt. -**Deferred to the World / project-as-crate cycle**: **full** content-addressed -reproduction (#158, C18) — the `topology_hash` field landed cycle 0092; what remains is -re-deriving the FlatGraph from a stored manifest and a content-id that covers -structural-axis variants; retiring the pre-C24 +**Content-addressed reproduction shipped (cycle 0094, #158, C18)**: `topology_hash` +landed cycle 0092; re-deriving a member's FlatGraph from a stored manifest + the +content-addressed blueprint store (`aura reproduce`) shipped cycle 0094 (see C18 +Realization). What **remains** is a content-id that covers **structural-axis / whole-harness +variants** (the scaffolding is not yet blueprint-data) and the debug-name-in-id question +(#171/#170). Still deferred: retiring the pre-C24 hard-wired `aura-cli` harnesses (`HarnessKind`, `run_stage1_r`, `*_sweep_family`) once the project-as-crate layer lands (#159, paired with #157's data-authoring surface). **Out of the first cut's round-trippable set** (deliberate; fails clean as `UnknownNodeType`, never a silent wrong graph): recording diff --git a/docs/plans/0094-content-addressed-reproduction.md b/docs/plans/0094-content-addressed-reproduction.md deleted file mode 100644 index 7ee82da..0000000 --- a/docs/plans/0094-content-addressed-reproduction.md +++ /dev/null @@ -1,541 +0,0 @@ -# Content-addressed reproduction — iteration 1 (the heart) — Implementation Plan - -> **Parent spec:** `docs/specs/0094-content-addressed-reproduction.md` -> -> **For agentic workers:** REQUIRED SUB-SKILL: use the `implement` skill to run -> this plan. Steps use `- [ ]` checkboxes for tracking. - -**Goal:** Content-address a blueprint-sweep's topology and add `aura reproduce -`, which re-derives every persisted member bit-identically from the -stored blueprint (acceptance 2). - -**Architecture:** A dumb content-addressed bytes store in `aura-registry` -(`runs/blueprints/.json`, keyed by the `topology_hash` the manifest already -carries) is written once per sweep at the existing persist seam. The per-member -reduce-mode run is extracted from the sweep closure into a shared -`run_blueprint_member`, so `reproduce_family` re-runs the *identical* path and a -match is bit-identical by construction (C1). Reproduction reconstructs each -member's bootstrap point from its recorded params (the inverse of `zip_params`). - -**Tech Stack:** `aura-registry` (store), `aura-cli` (persist write, reproduce -path, dispatch arm), reusing `aura-engine::blueprint_serde` (untouched, C9). - -**Acceptance 1 (`graph introspect --content-id`) and 3 (Tier-1 id stability) are -DEFERRED to a follow-on plan 0094b — not in this iteration.** - ---- - -## Files this plan creates or modifies - -- Modify: `crates/aura-registry/src/lib.rs` — add `put_blueprint` / `get_blueprint` - / `blueprints_dir` to `impl Registry` (~42-97); a round-trip unit test in the - test module. -- Modify: `crates/aura-cli/src/main.rs` — extract `run_blueprint_member` from - `blueprint_sweep_family`'s member closure (~3017-3045); add the store write in - `run_blueprint_sweep` (~3063-3082); add `reproduce_family` / `reproduce_family_in` - / `point_from_params` / `ReproduceReport`; add the `["reproduce", id]` dispatch - arm (~3853); a reproduce unit test in the `#[cfg(test)] mod tests`. -- Test: `crates/aura-cli/tests/cli_run.rs` — an E2E driving the built binary: - `aura sweep … --name X` then `aura reproduce X`; plus a missing-family rejection. - ---- - -## Task 1: Content-addressed blueprint store (aura-registry) - -**Files:** -- Modify: `crates/aura-registry/src/lib.rs:42-97` (add three methods to `impl Registry`) -- Test: `crates/aura-registry/src/lib.rs` (test module, ~639+) - -- [ ] **Step 1: Write the failing test** - -Add to the `#[cfg(test)] mod tests` (use the existing `temp_path(name)` helper at ~639): - -```rust -#[test] -fn blueprint_store_round_trips_by_content_id() { - let reg = Registry::open(temp_path("blueprint_store_round_trip")); - let canonical = r#"{"format_version":1,"input_roles":[],"nodes":[],"edges":[]}"#; - reg.put_blueprint("deadbeef", canonical).expect("put"); - // exact bytes come back, keyed by content id - assert_eq!(reg.get_blueprint("deadbeef").expect("get"), Some(canonical.to_string())); - // an absent id is Ok(None), not an error (treat-as-empty discipline) - assert_eq!(reg.get_blueprint("never-written").expect("get"), None); -} -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test -p aura-registry blueprint_store_round_trips_by_content_id` -Expected: FAIL — compile error (`no method named put_blueprint`). - -- [ ] **Step 3: Write the store methods** - -Add to `impl Registry` (the block at ~42-97). `RegistryError::Io` already exists -(491-504) with `impl From` (531-535), so `?` auto-maps: - -```rust - /// The content-addressed blueprint store dir — a sibling of the runs store, - /// `.with_file_name("blueprints")`. A topology is stored once, - /// keyed by its content id (`topology_hash`), so a whole sweep family's - /// members share one stored blueprint (C18 tiny manifest; C11/C12 dedup). - fn blueprints_dir(&self) -> std::path::PathBuf { - self.path.with_file_name("blueprints") - } - - /// Write-once content-addressed put: `blueprints/.json` = `canonical_json`. - /// Idempotent — the same content id always addresses identical canonical bytes, - /// so a repeated write re-writes identical content. The registry does NOT - /// verify `sha256(bytes) == hash` (no `sha2` dep here): the caller owns the - /// hash, and reproduction's bit-identical metric compare is the integrity check. - pub fn put_blueprint(&self, hash: &str, canonical_json: &str) -> Result<(), RegistryError> { - let dir = self.blueprints_dir(); - std::fs::create_dir_all(&dir)?; - std::fs::write(dir.join(format!("{hash}.json")), canonical_json)?; - Ok(()) - } - - /// Read a stored blueprint by content id; `Ok(None)` if absent — the same - /// treat-as-empty discipline `load` applies to a missing runs store. - pub fn get_blueprint(&self, hash: &str) -> Result, RegistryError> { - match std::fs::read_to_string(self.blueprints_dir().join(format!("{hash}.json"))) { - Ok(s) => Ok(Some(s)), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), - Err(e) => Err(RegistryError::Io(e)), - } - } -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cargo test -p aura-registry blueprint_store_round_trips_by_content_id` -Expected: PASS. - ---- - -## Task 2: Extract `run_blueprint_member` (behaviour-preserving) - -**Files:** -- Modify: `crates/aura-cli/src/main.rs:2983-3046` (`blueprint_sweep_family` — extract its member closure) - -The member closure at 3017-3045 is lifted verbatim into a named fn so reproduction -re-runs the identical reduce-mode path. NO behaviour change: the existing keystone -`blueprint_sweep_member_equals_single_run_and_shares_topology_hash` must stay green. - -- [ ] **Step 1: Add the extracted function** - -Insert immediately above `blueprint_sweep_family` (~2982). `ParamSpec`, `Cell`, -`Source` are already in scope in this file (used by the closure today): - -```rust -/// Run one bootstrapped member of a loaded-signal sweep: the reduce-mode path -/// (`SeriesReducer` folds eq/ex, `GatedRecorder` retains the gated R rows — the -/// O(cycles)→O(trades) fold), shared by the live sweep AND reproduction so a -/// reproduced member re-derives bit-identically (C1). `signal` is a freshly -/// reloaded blueprint (`Composite` is `!Clone`); `point` is the member's bound -/// cells; `space` gives the by-name manifest params; `topo` the shared signal hash. -#[allow(clippy::too_many_arguments)] -fn run_blueprint_member( - signal: Composite, - point: &[Cell], - space: &[ParamSpec], - sources: Vec>, - window: (Timestamp, Timestamp), - seed: u64, - pip: f64, - topo: &str, -) -> RunReport { - let (tx_eq, rx_eq) = mpsc::channel(); - let (tx_ex, rx_ex) = mpsc::channel(); - let (tx_r, rx_r) = mpsc::channel(); - let (tx_req, _rx_req) = mpsc::channel(); - let mut h = wrap_stage1r(signal, tx_eq, tx_ex, tx_r, tx_req, false, true, None) - .bootstrap_with_cells(point) - .expect("member bootstraps (point kind-checked against param_space)"); - h.run(sources); - let named = zip_params(space, point); // by-name params for the manifest record - let mut manifest = sim_optimal_manifest(named, window, seed, pip); - manifest.broker = stage1_r_broker_label(pip); - manifest.topology_hash = Some(topo.to_string()); - let r_rows: Vec<(Timestamp, Vec)> = rx_r.try_iter().collect(); - let (total_pips, max_drawdown) = rx_eq - .try_iter() - .next() - .map(|(_, row)| (row[0].as_f64(), row[1].as_f64())) - .unwrap_or((0.0, 0.0)); - let bias_sign_flips = - rx_ex.try_iter().next().map(|(_, row)| row[2].as_i64() as u64).unwrap_or(0); - let mut m = RunMetrics { total_pips, max_drawdown, bias_sign_flips, r: None }; - m.r = Some(summarize_r(&r_rows, &[])); - RunReport { manifest, metrics: m } -} -``` - -- [ ] **Step 2: Rewrite the sweep closure to call it** - -Replace the body of `binder.sweep(|point| { … })` at 3017-3045 with: - -```rust - binder.sweep(|point| { - // fresh per-member graph (Composite is !Clone, reload per member) run through - // the shared reduce-mode member path — the same fn reproduction re-runs. - run_blueprint_member(reload(doc), point, &space, data.run_sources(), window, 0, pip, &topo) - }) -``` - -- [ ] **Step 3: Build** - -Run: `cargo build -p aura-cli` -Expected: 0 errors. - -- [ ] **Step 4: Verify the extraction preserved behaviour** - -Run: `cargo test -p aura-cli --bin aura blueprint_sweep_member_equals_single_run_and_shares_topology_hash` -Expected: PASS (the reduce member still equals the full single run at fast=2/slow=4 — the extraction changed no behaviour). - ---- - -## Task 3: Store the canonical blueprint at the persist seam - -**Files:** -- Modify: `crates/aura-cli/src/main.rs:3063-3082` (`run_blueprint_sweep`) - -- [ ] **Step 1: Add the store write before `append_family`** - -Replace `run_blueprint_sweep`'s body (3064-3081) with (the `let _ = persist;` line and -the append/print tail are unchanged; only the `reg` binding + the put are new): - -```rust - let _ = persist; // reserved for the deferred per-member trace path; the family record below is unconditional - let family = blueprint_sweep_family(doc, axes, &data).unwrap_or_else(|e| { - eprintln!("aura: {e:?}"); - std::process::exit(2); - }); - let reg = default_registry(); - // Store the canonical blueprint ONCE, keyed by the family's shared topology_hash — - // exactly the bytes whose SHA256 the members carry (#164 byte-canonical, round-trip - // idempotent). One stored topology per family (C18/C11/C12). - let topo = family.points[0] - .report - .manifest - .topology_hash - .clone() - .expect("a blueprint sweep stamps every member's topology_hash"); - let canonical = blueprint_to_json( - &blueprint_from_json(doc, &|t| std_vocabulary(t)) - .expect("doc parse-validated at the dispatch boundary"), - ) - .expect("a loaded blueprint re-serializes"); - reg.put_blueprint(&topo, &canonical).unwrap_or_else(|e| { - eprintln!("aura: {e}"); - std::process::exit(2); - }); - // Record the family unconditionally (C18/C21 lineage), exactly like `run_sweep`. - let id = match reg.append_family(name, FamilyKind::Sweep, &sweep_member_reports(&family)) { - Ok(id) => id, - Err(e) => { - eprintln!("aura: {e}"); - std::process::exit(2); - } - }; - for pt in &family.points { - println!("{}", family_member_line(&id, &pt.report)); - } -``` - -- [ ] **Step 2: Build + verify existing sweep behaviour unperturbed** - -Run: `cargo build -p aura-cli` -Expected: 0 errors. - -- [ ] **Step 3: Verify the existing blueprint-sweep E2E still passes** - -Run: `cargo test -p aura-cli --test cli_run aura_sweep_loads_a_blueprint_and_persists_a_sweep_family` -Expected: PASS (the store write is additive + stdout-silent; the persisted family + printed lines are unchanged). - ---- - -## Task 4: Reproduce path + dispatch arm - -**Files:** -- Modify: `crates/aura-cli/src/main.rs` (add `ReproduceReport`, `point_from_params`, - `reproduce_family_in`, `reproduce_family`, the `["reproduce", id]` arm ~3853) -- Test: `crates/aura-cli/src/main.rs` (`#[cfg(test)] mod tests`) - -- [ ] **Step 1: Write the failing test** - -Add to `#[cfg(test)] mod tests` (mirrors the keystone setup at 4986-4997, adds a temp -registry + the persist + reproduce round-trip). The grid includes fast=2/slow=4, the -open-at-end param (`n_open_at_end=1`): - -```rust -#[test] -fn reproduce_family_re_derives_every_member_bit_identically() { - // a unique temp runs store so the on-disk family + blueprint store do not collide. - let dir = std::env::temp_dir().join(format!("aura-repro-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).expect("temp dir"); - let reg = Registry::open(dir.join("runs.jsonl")); - - let open = stage1_signal(None, None); - let doc = blueprint_to_json(&open).expect("serializes"); - let data = DataSource::Synthetic; - // 2x grid over slow.length {4,6} at fast=2 — slow=4 is the open-at-end member. - let axes = vec![ - ("stage1_signal.fast.length".to_string(), vec![Scalar::i64(2)]), - ("stage1_signal.slow.length".to_string(), vec![Scalar::i64(4), Scalar::i64(6)]), - ]; - let family = blueprint_sweep_family(&doc, &axes, &data).expect("axes resolve"); - - // persist exactly as run_blueprint_sweep does: store the blueprint, append the family. - let topo = family.points[0].report.manifest.topology_hash.clone().expect("topo"); - let canonical = - blueprint_to_json(&blueprint_from_json(&doc, &|t| std_vocabulary(t)).unwrap()).unwrap(); - reg.put_blueprint(&topo, &canonical).expect("store blueprint"); - let id = reg - .append_family("repro", FamilyKind::Sweep, &sweep_member_reports(&family)) - .expect("append"); - - // reproduce: every member re-derives bit-identically (incl the open-at-end member). - let rep = reproduce_family_in(®, &id, &data); - assert_eq!(rep.outcomes.len(), 2, "two members reproduced"); - assert!( - rep.outcomes.iter().all(|(_, ok)| *ok), - "every member re-derives bit-identically: {:?}", - rep.outcomes - ); - - let _ = std::fs::remove_dir_all(&dir); -} -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test -p aura-cli --bin aura reproduce_family_re_derives_every_member_bit_identically` -Expected: FAIL — compile error (`cannot find function reproduce_family_in` / `ReproduceReport`). - -- [ ] **Step 3: Write the reproduce path** - -Add near `runs_family` (~2470). `group_families`, `load_family_members`, `Cell`, -`ParamSpec`, `Scalar`, `topology_hash`, `blueprint_from_json`, `std_vocabulary` are all -in scope: - -```rust -/// The outcome of reproducing one persisted family: per member, whether its re-run -/// metrics are bit-identical to the stored metrics (C1). -struct ReproduceReport { - outcomes: Vec<(String, bool)>, -} - -/// Reconstruct a member's bootstrap point from its recorded named params — the inverse -/// of `zip_params(space, point)`. Walks the reloaded signal's `param_space` in order -/// (deterministic for the same blueprint) and reads each knob's value from the manifest. -fn point_from_params(space: &[ParamSpec], params: &[(String, Scalar)]) -> Vec { - space - .iter() - .map(|ps| { - params - .iter() - .find(|(n, _)| n == &ps.name) - .map(|(_, s)| s.cell()) - .unwrap_or_else(|| panic!("manifest is missing param {}", ps.name)) - }) - .collect() -} - -/// Re-derive every member of a persisted sweep family from the content-addressed store -/// and compare to the stored result, against an explicit registry (testable seam). -fn reproduce_family_in(reg: &Registry, id: &str, data: &DataSource) -> ReproduceReport { - let members = reg.load_family_members().unwrap_or_else(|e| { - eprintln!("aura: {e}"); - std::process::exit(2); - }); - let Some(family) = group_families(members).into_iter().find(|f| f.id == id) else { - // reproduce is an action, not a lookup: an unknown id is a hard error (distinct - // from `runs family `'s treat-as-empty exit 0). - eprintln!("aura: no such family '{id}'"); - std::process::exit(2); - }; - let pip = data.pip_size(); - let window = data.full_window(); - let mut outcomes = Vec::new(); - for member in &family.members { - let stored = &member.report; - let hash = stored.manifest.topology_hash.clone().unwrap_or_else(|| { - eprintln!("aura: family member has no topology_hash; not a generated run"); - std::process::exit(2); - }); - let doc = reg - .get_blueprint(&hash) - .unwrap_or_else(|e| { - eprintln!("aura: {e}"); - std::process::exit(2); - }) - .unwrap_or_else(|| { - eprintln!("aura: blueprint {hash} missing from store"); - std::process::exit(2); - }); - let signal = blueprint_from_json(&doc, &|t| std_vocabulary(t)).unwrap_or_else(|e| { - eprintln!("aura: stored blueprint {hash} does not parse: {e:?}"); - std::process::exit(2); - }); - let space = signal.param_space(); - let point = point_from_params(&space, &stored.manifest.params); - let label = stored - .manifest - .params - .iter() - .map(|(n, v)| format!("{n}={v:?}")) - .collect::>() - .join(", "); - let rerun = run_blueprint_member( - signal, - &point, - &space, - data.run_sources(), - window, - stored.manifest.seed, - pip, - &hash, - ); - outcomes.push((label, rerun.metrics == stored.metrics)); - } - ReproduceReport { outcomes } -} - -/// `aura reproduce `: re-derive a persisted sweep family from the -/// content-addressed blueprint store and verify each member reproduces bit-identically -/// (C18 "re-derives full results on demand"). Synthetic data this cycle (deterministic); -/// recorded-dataset reproduction rides the DataServer seam (#124). -fn reproduce_family(id: &str) { - let rep = reproduce_family_in(&default_registry(), id, &DataSource::Synthetic); - let total = rep.outcomes.len(); - let ok = rep.outcomes.iter().filter(|(_, b)| *b).count(); - for (label, identical) in &rep.outcomes { - let verdict = if *identical { "bit-identical" } else { "DIVERGED" }; - println!("{id} member {label} reproduced: {verdict}"); - } - println!("reproduced {ok}/{total} members bit-identically"); - if ok != total { - std::process::exit(1); - } -} -``` - -- [ ] **Step 4: Add the dispatch arm** - -In the top-level `match` (~3736), beside `["runs", "family", id]` (~3853), add: - -```rust - ["reproduce", id] => reproduce_family(id), -``` - -- [ ] **Step 5: Run test to verify it passes** - -Run: `cargo test -p aura-cli --bin aura reproduce_family_re_derives_every_member_bit_identically` -Expected: PASS (every member, incl the open-at-end one, re-derives bit-identically). - ---- - -## Task 5: End-to-end CLI coverage - -**Files:** -- Test: `crates/aura-cli/tests/cli_run.rs` (append two tests, mirroring - `aura_sweep_loads_a_blueprint_and_persists_a_sweep_family` at 3269-3288) - -- [ ] **Step 1: Write the E2E tests** - -Append to `crates/aura-cli/tests/cli_run.rs` (`BIN` at :8, `temp_cwd` at :13): - -```rust -/// E2E (acc 2): persist a blueprint sweep, then `aura reproduce ` re-derives every -/// member bit-identically from the content-addressed store and exits 0. -#[test] -fn aura_reproduce_re_derives_a_persisted_sweep_bit_identically() { - let cwd = temp_cwd("reproduce_roundtrip"); - let fixture = format!("{}/tests/fixtures/stage1_signal_open.json", env!("CARGO_MANIFEST_DIR")); - - let sweep = std::process::Command::new(BIN) - .args([ - "sweep", - &fixture, - "--axis", - "stage1_signal.fast.length=2", - "--axis", - "stage1_signal.slow.length=4,6", - "--name", - "smacross", - ]) - .current_dir(&cwd) - .output() - .expect("run aura sweep"); - assert!(sweep.status.success(), "sweep ok: {}", String::from_utf8_lossy(&sweep.stderr)); - - let repro = std::process::Command::new(BIN) - .args(["reproduce", "smacross-1"]) - .current_dir(&cwd) - .output() - .expect("run aura reproduce"); - let stdout = String::from_utf8_lossy(&repro.stdout); - assert!(repro.status.success(), "reproduce exits 0: {}", String::from_utf8_lossy(&repro.stderr)); - assert!( - stdout.contains("reproduced 2/2 members bit-identically"), - "every member re-derives: {stdout}" - ); - - let _ = std::fs::remove_dir_all(&cwd); -} - -/// E2E (negative): `aura reproduce ` is a hard error (exit non-zero + named -/// cause on stderr), distinct from `runs family `'s treat-as-empty exit 0. -#[test] -fn aura_reproduce_rejects_an_unknown_family() { - let cwd = temp_cwd("reproduce_unknown"); - let out = std::process::Command::new(BIN) - .args(["reproduce", "no-such-family-1"]) - .current_dir(&cwd) - .output() - .expect("run aura reproduce"); - assert!(!out.status.success(), "unknown family exits non-zero"); - assert!( - String::from_utf8_lossy(&out.stderr).contains("no such family 'no-such-family-1'"), - "names the cause: {}", - String::from_utf8_lossy(&out.stderr) - ); - let _ = std::fs::remove_dir_all(&cwd); -} -``` - -- [ ] **Step 2: Build the binary + run the E2E** - -Run: `cargo test -p aura-cli --test cli_run aura_reproduce_re_derives_a_persisted_sweep_bit_identically` -Expected: PASS. - -- [ ] **Step 3: Run the negative E2E** - -Run: `cargo test -p aura-cli --test cli_run aura_reproduce_rejects_an_unknown_family` -Expected: PASS. - -- [ ] **Step 4: Full workspace gate** - -Run: `cargo test --workspace` -Expected: PASS (no regressions). - -Run: `cargo clippy --workspace --all-targets -- -D warnings` -Expected: clean. - ---- - -## Notes for the implementer - -- **`member.report` hop:** a grouped family's members are `FamilyRunRecord`; the - payload is `member.report` (a `RunReport`), so fields are `member.report.manifest.*` - / `member.report.metrics` (the spec pseudo wrote `member.manifest.*` for brevity). -- **`RMetrics` equality excludes `trade_rs`** (hand-written `PartialEq`, - aura-analysis/src/lib.rs:111-124; `trade_rs` is `#[serde(skip)]`). A stored member's - empty `trade_rs` vs. a re-run's populated one therefore does NOT break - `rerun.metrics == stored.metrics` — this is what makes the compare well-defined. -- **`Scalar::cell()`** (aura-core/src/scalar.rs:89) is the `Scalar → Cell` conversion for - `point_from_params`; the forward `zip_params` lives at aura-core/src/node.rs:83. -- **Seed:** the sweep records seed 0 (`run_blueprint_member(..., 0, ...)` from the sweep - call site); reproduction threads `stored.manifest.seed` (= 0) through the same fn. -- Do NOT add `graph introspect --content-id` or the Tier-1 pin here — they are plan 0094b. diff --git a/docs/specs/0094-content-addressed-reproduction.md b/docs/specs/0094-content-addressed-reproduction.md deleted file mode 100644 index d84161c..0000000 --- a/docs/specs/0094-content-addressed-reproduction.md +++ /dev/null @@ -1,285 +0,0 @@ -# Content-addressed reproduction of a generated run — Design Spec - -**Date:** 2026-07-01 -**Status:** Draft — awaiting user spec review -**Authors:** orchestrator + Claude - -## Goal - -Restore the C18 reproducibility guarantee for **generated** runs. Since cycle 0093 a -sweep member's topology is a *generated data value* (a `Composite` loaded from a blueprint -file, gridded over named axes), so `commit` alone no longer identifies the graph and the -member's manifest is no longer a complete re-derivation recipe — a C18 violation (#158). - -This cycle closes that gap by **content-addressing the topology**: the canonical blueprint -JSON is stored once, keyed by its `topology_hash` (already on the manifest since cycle -0092), and a reproduce path re-derives a persisted family's runs from the store and proves -each is bit-identical to the stored result (C1). The three #158 acceptance properties are -realized as tests; the user-facing capability is `aura reproduce `. - -The hash domain is **signal-only** this cycle — the fixed Stage-1-R scaffolding -(`wrap_stage1r`: broker / sinks / RiskExecutor / cost-graph) is not yet blueprint-data -(deliberately out of the round-trippable set, C24) so the content-id covers the signal and -the scaffolding stays commit-identified. The boundary is documented; whole-harness -content-addressing is deferred (#156 / #159 / project-as-crate). - -## Architecture - -Three layers, each at its correct home: - -1. **Content-addressed blueprint store (aura-registry).** The registry owns the on-disk - `runs/` layout (`runs.jsonl`, `families.jsonl` sibling, `runs/traces//`). It gains - a sibling content-addressed store `runs/blueprints/.json` and two dumb - bytes-by-key methods: `put_blueprint(hash, canonical_json)` (write-once: identical - content re-writes the identical bytes, a no-op in effect) and - `get_blueprint(hash) -> Option`. The registry does **not** parse, hash, or - validate the bytes — it stores and retrieves a String keyed by a String. No `sha2` - dependency enters the registry: the caller (the CLI) computes the key, and the - reproduce path's bit-identical metric comparison is itself the integrity check (a - corrupted blueprint re-derives different metrics → reported as a divergence). - -2. **Hash + canonical bytes + reproduce (aura-cli).** The CLI already owns `topology_hash` - (`sha2` over `blueprint_to_json`, off the frozen engine, invariant 8). On a blueprint - sweep it writes the canonical blueprint once (the bytes whose SHA256 *is* the shared - `topology_hash`); the reproduce path reads a stored family, loads each member's - blueprint by hash, reconstructs the member's sweep point from its recorded params, - re-bootstraps through the cycle-1 `wrap_stage1r` seam, re-runs, and compares. - -3. **Cross-path content-id (aura graph introspect).** A minimal read-only addition - `aura graph introspect --content-id` prints the SHA256 of the canonical blueprint an - op-script (#157) builds — making "do these two authoring paths describe the same - topology?" answerable from the CLI, and giving acceptance (1) its surface. - -C9 holds throughout: the engine is untouched (`blueprint_to_json` / `blueprint_from_json` -already exist in `aura-engine::blueprint_serde`); the new store and reproduce logic live in -the research-side crates that already depend on the engine. - -## Concrete code shapes - -### Worked user-facing capability — `aura reproduce ` - -The research user persisted a sweep family and wants to verify it still reproduces from its -manifest (the C18 "re-derives full results on demand" promise): - -```console -$ aura sweep stage1_signal_open.json --axis stage1_signal.slow.length=4,6,8,10 --name smacross -smacross-1 stage1_signal.slow.length=4 expectancy_r=1.2710 n_trades=3 -smacross-1 stage1_signal.slow.length=6 expectancy_r=... n_trades=... -... -# topology stored once at runs/blueprints/.json; 4 member manifests share that hash. - -$ aura reproduce smacross-1 -smacross-1 member stage1_signal.slow.length=4 reproduced: bit-identical -smacross-1 member stage1_signal.slow.length=6 reproduced: bit-identical -smacross-1 member stage1_signal.slow.length=8 reproduced: bit-identical -smacross-1 member stage1_signal.slow.length=10 reproduced: bit-identical -reproduced 4/4 members bit-identically from runs/blueprints/.json -$ echo $? -0 -# a divergence (or a missing stored blueprint) prints the member + exits non-zero. -``` - -This is the empirical evidence for the feature-acceptance criterion: the capability a -research user reaches for, built on the manifest + the content-addressed store + the commit -as the complete re-derivation recipe. - -### Store write at the sweep persist seam (aura-cli `run_blueprint_sweep`, ~main.rs:3063) - -Before (today — the family is persisted, the topology is not stored): - -```rust -fn run_blueprint_sweep(doc: &str, axes: &[(String, Vec)], name: &str, persist: bool, data: DataSource) { - let _ = persist; - let family = blueprint_sweep_family(doc, axes, &data).unwrap_or_else(|e| { eprintln!("aura: {e:?}"); std::process::exit(2); }); - let id = match default_registry().append_family(name, FamilyKind::Sweep, &sweep_member_reports(&family)) { /* … */ }; - for pt in &family.points { println!("{}", family_member_line(&id, &pt.report)); } -} -``` - -After (the shared topology is content-addressed once, beside the family record): - -```rust -fn run_blueprint_sweep(doc: &str, axes: &[(String, Vec)], name: &str, persist: bool, data: DataSource) { - let _ = persist; - let family = blueprint_sweep_family(doc, axes, &data).unwrap_or_else(|e| { eprintln!("aura: {e:?}"); std::process::exit(2); }); - let reg = default_registry(); - // store the canonical blueprint once, keyed by the family's shared topology_hash — - // exactly the bytes whose SHA256 the members already carry (#164 byte-canonical form). - let topo = family.points[0].report.manifest.topology_hash.clone() - .expect("a blueprint sweep stamps every member's topology_hash"); - let canonical = blueprint_to_json(&blueprint_from_json(doc, &|t| std_vocabulary(t)) - .expect("doc parse-validated at the dispatch boundary")) - .expect("a loaded blueprint re-serializes"); - reg.put_blueprint(&topo, &canonical).unwrap_or_else(|e| { eprintln!("aura: {e}"); std::process::exit(2); }); - let id = match reg.append_family(name, FamilyKind::Sweep, &sweep_member_reports(&family)) { /* … unchanged … */ }; - for pt in &family.points { println!("{}", family_member_line(&id, &pt.report)); } -} -``` - -### Content-addressed store (aura-registry, new methods on `Registry`) - -```rust -impl Registry { - /// The content-addressed blueprint store dir, a sibling of the runs store: - /// `.with_file_name("blueprints")`. - fn blueprints_dir(&self) -> PathBuf { self.path.with_file_name("blueprints") } - - /// Write-once content-addressed put: `runs/blueprints/.json` = canonical_json. - /// Idempotent — the same hash always addresses identical bytes, so an overwrite is a - /// no-op in effect. The registry does not verify `sha256(bytes) == hash`; the caller - /// owns the hash, and reproduction's bit-identical compare is the integrity check. - pub fn put_blueprint(&self, hash: &str, canonical_json: &str) -> Result<(), RegistryError> { - let dir = self.blueprints_dir(); - std::fs::create_dir_all(&dir).map_err(/* RegistryError::Io */)?; - std::fs::write(dir.join(format!("{hash}.json")), canonical_json).map_err(/* … */)?; - Ok(()) - } - - /// Read a stored blueprint by content id; `Ok(None)` if absent (treat-as-empty - /// discipline, like `load` on a missing store). - pub fn get_blueprint(&self, hash: &str) -> Result, RegistryError> { - match std::fs::read_to_string(self.blueprints_dir().join(format!("{hash}.json"))) { - Ok(s) => Ok(Some(s)), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), - Err(e) => Err(/* RegistryError::Io */), - } - } -} -``` - -### Reproduce path (aura-cli, new) - -```rust -/// Re-derive every member of a persisted sweep family from the content-addressed store and -/// compare to the stored result. Determinism (C1) makes a match bit-identical. -fn reproduce_family(id: &str, data: &DataSource) -> ReproduceReport { - let reg = default_registry(); - let family = group_families(reg.load_family_members().expect("load")) - .into_iter().find(|f| f.id == id) - .unwrap_or_else(|| { eprintln!("aura: no such family '{id}'"); std::process::exit(2); }); - - let mut outcomes = Vec::new(); - for member in &family.members { - let hash = member.manifest.topology_hash.as_deref() - .expect("a generated member carries a topology_hash"); - let doc = reg.get_blueprint(hash).expect("store read") - .unwrap_or_else(|| { eprintln!("aura: blueprint {hash} missing from store"); std::process::exit(2); }); - // reconstruct the sweep point: param_space order of the reloaded signal, values - // read from the member's recorded named params (the inverse of zip_params). - let signal = blueprint_from_json(&doc, &|t| std_vocabulary(t)).expect("stored blueprint parses"); - let point = point_from_params(&signal, &member.manifest.params); // Vec in param_space order - let rerun = run_signal_point(signal, &point, data, member.manifest.seed); // wrap_stage1r + bootstrap + run, reduce-mode - outcomes.push((member_label(member), rerun.metrics == member.metrics)); - } - ReproduceReport { id: id.to_string(), outcomes } -} -``` - -`point_from_params` walks the reloaded signal's `param_space()` and, for each knob, reads -the value the member's manifest recorded under that knob's name, yielding the `Vec` -`bootstrap_with_cells` consumes — the inverse of the `zip_params(&space, point)` the sweep -used to record them. - -### Cross-path content-id surface (aura graph introspect, ~main.rs introspect arm) - -```rust -// `aura graph introspect --content-id` over an op-script on stdin: build the blueprint -// (the #157 replay), print the SHA256 of its canonical form — the same hash `aura run` -// stamps as topology_hash. Reuses topology_hash(&composite); no new hashing. -["graph", "introspect", "--content-id"] => { - let blueprint = build_from_op_script(read_stdin())?; // existing #157 replay → Composite - println!("{}", topology_hash(&blueprint)); -} -``` - -## Components - -- **`Registry::put_blueprint` / `get_blueprint` / `blueprints_dir`** (aura-registry) — the - content-addressed bytes store under `runs/blueprints/`. Dumb, no engine-blueprint parse, - no `sha2`. -- **Store write in `run_blueprint_sweep`** (aura-cli) — one canonical-blueprint write per - sweep, keyed by the family's shared `topology_hash`. -- **`reproduce_family` + `point_from_params` + the `aura reproduce ` dispatch arm** - (aura-cli) — load family → load blueprint by hash → reconstruct point → re-run → compare. -- **`graph introspect --content-id`** (aura-cli) — the op-script content-id surface for - acceptance (1). - -## Data flow - -Persist: `aura sweep ` → `blueprint_sweep_family` (members stamped with the shared -`topology_hash`) → `run_blueprint_sweep` writes `runs/blueprints/.json` once → -`append_family` writes the member records to `families.jsonl`. - -Reproduce: `aura reproduce ` → `group_families` finds the family → per member: read -`topology_hash` → `get_blueprint(hash)` → `blueprint_from_json` → `point_from_params` → -`wrap_stage1r` + `bootstrap_with_cells(point)` + `run` (reduce-mode, exactly the sweep -member path) → `summarize_r` → compare re-run `metrics` to the stored member `metrics`. - -The manifest carries `params`, `window`, `seed`, `topology_hash`; the data source is -synthetic (deterministic) for this cycle — recorded-dataset reproduction rides the -DataServer seam (#124), out of scope. - -## Error handling - -- **Missing stored blueprint** (`get_blueprint` → `None`): the family was recorded before - the store existed, or the store dir was deleted — print `blueprint missing from - store` and exit non-zero (refuse-don't-guess, C10/C18). Never re-run against a guessed - topology. -- **Unknown family id**: `aura reproduce ` with no matching family prints `no such - family ''` and exits non-zero (a deliberate non-zero, distinct from the treat-as-empty - `runs family ` *lookup* — reproduce is an action that needs a real target). -- **Divergence** (`rerun.metrics != stored`): report the offending member and exit - non-zero. This is the integrity check; on synthetic data a divergence is a real bug, not - an expected outcome. -- **Store write failure** (I/O): propagate as `RegistryError`, surfaced as `aura: ` + - exit non-zero at the persist seam. - -## Testing strategy - -- **acc 2 (the heart) — reproduce is bit-identical, including open-at-end.** A test - persists a blueprint sweep whose grid includes the **open-at-end** param (fast=2/slow=4 is - open-at-end, `n_open_at_end=1`), then `reproduce_family` re-derives every member and - asserts `rerun.metrics == stored.metrics` for each. This exercises the open-at-end path - end-to-end through the real reproduce loop — store round-trip + C1 determinism: a stored - member re-derives bit-identically (a reduce-vs-reduce re-run, the same path the sweep - recorded). The reduce(`GatedRecorder`) vs full(`Recorder`) R-equivalence the cycle-0093 - implementer flagged is a *separate* property, already pinned by the existing keystone - `blueprint_sweep_member_equals_single_run_and_shares_topology_hash` (a reduce member == - a full single run at fast=2/slow=4, an open-at-end param) plus two unit tests — see #166; - this cycle adds no new claim there, it reuses the open-at-end param for thorough reproduce - coverage. -- **acc 2 — round-trip store.** A unit test: `put_blueprint(h, bytes)` then - `get_blueprint(h)` returns `Some(bytes)` byte-identically; `get_blueprint(absent)` is - `Ok(None)`. -- **acc 1 — two command paths, same content id.** A test builds the SMA-cross signal via - the op-script path (`graph introspect --content-id`, or hashing `graph build` output) and - via `stage1_signal`/`topology_hash`, and asserts the two content ids are equal for the - same topology and differ for a different one (e.g. fast=2 vs fast=3). -- **acc 3 — Tier-1 stability of the content id.** A test pins that `topology_hash` is - unchanged when a Tier-1 optional field the blueprint does not use is added to the format - — composing with `unknown_optional_field_is_tolerated_byte_identically` (#156): the - canonical omit-defaults form omits an absent optional, so the hashed bytes (and the id) - are byte-identical. -- **E2E (CLI):** `aura sweep --name X` then `aura reproduce X` over the built binary — - asserts the `reproduced N/N members bit-identically` line and exit 0; a missing-store and - an unknown-id path each exit non-zero with the named cause on stderr. - -## Acceptance criteria - -1. **Two command paths that build the same topology produce the same content id.** The - op-script path and `aura run`'s `stage1_signal` agree on `topology_hash` for the same - signal, and disagree for a different one. (`graph introspect --content-id` exposes it.) -2. **A serialized manifest re-derives the same FlatGraph and a bit-identical run (C1).** - `aura reproduce ` loads each member's blueprint from the content-addressed - store by its manifest's `topology_hash`, re-bootstraps with the member's params / window - / seed, re-runs, and the re-run metrics are bit-identical to the stored metrics — - including the open-at-end param. -3. **A Tier-1 format addition the blueprint does not use leaves its content id unchanged.** - An absent optional does not enter the canonical form, so `topology_hash` is byte-stable - across the addition. -4. **Boundary documented.** The content-id covers the **signal** only; the Stage-1-R - scaffolding stays commit-identified this cycle (recorded in the ledger), with - whole-harness content-addressing deferred. -5. **Invariants preserved.** Engine untouched (C9); the store + hash are off the frozen - engine (invariant 8); reproduction is deterministic on synthetic data (C1); - refuse-don't-guess on a missing stored blueprint (C10/C18).