feat(registry): index-first identity resolution with self-repairing sidecar

Replace the per-call O(store) scan in find_blueprint_by_identity with a persistent identity-id -> content-id sidecar index (blueprint_identity_index.jsonl, sibling of runs.jsonl). Lookups consult the index first and VERIFY every hit by loading that one blueprint under the current roster and recomputing its identity id -- the index is a cache, never an oracle, so results stay scan-identical under roster drift, store surgery, or index corruption (up to the same-identity-twin choice, unspecified in both paths and documented on the fn). Any miss or failed verification runs the old scan as a full-store repair pass: no early return, best-effort append of every absent-or-different mapping, last line wins so later repairs heal stale lines. No write-path, engine, or caller changes; a pre-index store backfills itself on the first miss; a read-only store keeps scanning as before. Decided against put-time index maintenance: it would need a roster-free doc-level identity function whose equivalence to the loaded-composite path no green test ratifies (the composite path canonicalizes structurally on load); the repair path is the load-bearing mechanism either way. Decision log: the issue's comments. Verification: 7 new aura-registry tests (backfill, verified-hit-no-walk via a poison-dir probe, one-pass legacy backfill, stale-line healing, unloadable-under-roster preservation, garbage tolerance) plus a CLI e2e (campaign validate resolving an identity ref cold then warm); workspace suite 1346 passed / 0 failed; clippy -D warnings clean; doc build clean. closes #191
2026-07-17 14:13:48 +02:00
parent bfb8648925
commit 88a1c28954
2 changed files with 454 additions and 6 deletions
@@ -416,6 +416,164 @@ fn campaign_validate_in_project_reports_referential_tier_end_to_end() {
    // drop, at the end of this scope — including on a mid-test panic.
 }

+/// Property (#191, index-first identity lookup at the CLI seam): a strategy
+/// referenced by `identity_id` (never previously exercised end to end — every
+/// prior campaign e2e fixture used `content_id`) resolves through `campaign
+/// validate`'s referential tier exactly like a `content_id` ref. The FIRST
+/// resolution has no sidecar index yet, so `Registry::find_blueprint_by_identity`
+/// falls back to the store scan and backfills `blueprint_identity_index.jsonl`
+/// as a repair pass; the SECOND resolution goes through that now-present index
+/// (a verified hit). Both calls must answer identically — this is the E2E
+/// manifestation of "index-first lookup with verify-on-hit and repair-walk":
+/// if the backfilled index or its verify-on-hit read disagreed with the scan,
+/// only the second call would regress.
+#[test]
+fn campaign_validate_resolves_identity_ref_via_index_first_lookup_then_index_hit() {
+    let (dir, _fixture) = fresh_project();
+    let runs_dir = dir.join("runs");
+    std::fs::remove_dir_all(&runs_dir).ok();
+    let _cleanup = ScratchGuard(vec![
+        ScratchPath::Dir(runs_dir.clone()),
+        ScratchPath::File(dir.join("seed.process.json")),
+        ScratchPath::File(dir.join("identity-ok.campaign.json")),
+    ]);
+
+    // Seed one real, content-addressed blueprint (mirrors the content_id
+    // sibling test above).
+    let closed_bp = format!("{}/examples/r_sma.json", env!("CARGO_MANIFEST_DIR"));
+    let (sweep_out, sweep_code) = run_code_in(
+        &dir,
+        &[
+            "sweep",
+            &closed_bp,
+            "--axis",
+            "sma_signal.fast.length=2,4",
+            "--axis",
+            "sma_signal.slow.length=8,16",
+            "--name",
+            "campaign-identity-seed",
+        ],
+    );
+    assert_eq!(sweep_code, Some(0), "seed sweep failed: {sweep_out}");
+    let bp_id = std::fs::read_dir(runs_dir.join("blueprints"))
+        .expect("blueprints dir")
+        .next()
+        .expect("one stored blueprint")
+        .expect("dir entry")
+        .path()
+        .file_stem()
+        .expect("stem")
+        .to_string_lossy()
+        .into_owned();
+    let bp_path = runs_dir.join("blueprints").join(format!("{bp_id}.json"));
+
+    // The identity id of the SAME stored bytes — `--content-id FILE
+    // --identity-id` combined, over the store's own file so the computed
+    // identity id is guaranteed to match what the registry recomputes on a
+    // lookup (no risk of drift from re-deriving it off a differently-bound
+    // copy of the blueprint).
+    let (id_out, id_code) =
+        run_code_in(&dir, &["graph", "introspect", "--content-id", bp_path.to_str().expect("utf8 path"), "--identity-id"]);
+    assert_eq!(id_code, Some(0), "stdout/stderr: {id_out}");
+    let mut id_lines = id_out.lines();
+    let printed_content_id = id_lines.next().expect("content id line");
+    assert_eq!(printed_content_id, bp_id, "content id of the stored file matches its store key");
+    let identity_id = id_lines.next().expect("identity id line").to_string();
+
+    let exec_process = PROCESS_DOC
+        .replacen(
+            "\"select\": \"plateau:worst\", \"deflate\": true",
+            "\"select\": \"argmax\", \"deflate\": false",
+            1,
+        )
+        .replacen("\"overfit_probability\", \"cmp\": \"lt\", \"value\": 0.1", "\"win_rate\", \"cmp\": \"lt\", \"value\": 1.1", 1);
+    write_doc(&dir, "seed.process.json", &exec_process);
+    let (proc_out, proc_code) = run_code_in(&dir, &["process", "register", "seed.process.json"]);
+    assert_eq!(proc_code, Some(0), "seed process register failed: {proc_out}");
+    let proc_id = proc_out
+        .lines()
+        .find(|l| l.starts_with("registered process "))
+        .expect("register line")
+        .trim_start_matches("registered process ")
+        .split(' ')
+        .next()
+        .expect("id")
+        .to_string();
+
+    let campaign = format!(
+        r#"{{
+  "format_version": 1,
+  "kind": "campaign",
+  "name": "identity-ref-check",
+  "data": {{ "instruments": ["GER40"], "windows": [ {{ "from_ms": 1, "to_ms": 2 }} ] }},
+  "strategies": [ {{ "ref": {{ "identity_id": "{identity}" }},
+                    "axes": {{ "fast.length": {{ "kind": "I64", "values": [2] }},
+                              "slow.length": {{ "kind": "I64", "values": [6] }} }} }} ],
+  "process": {{ "ref": {{ "content_id": "{proc}" }} }},
+  "seed": 1,
+  "presentation": {{ "persist_taps": [], "emit": ["family_table"] }}
+}}"#,
+        identity = identity_id,
+        proc = proc_id
+    );
+    write_doc(&dir, "identity-ok.campaign.json", &campaign);
+
+    let sidecar = runs_dir.join("blueprint_identity_index.jsonl");
+    assert!(!sidecar.exists(), "no index sidecar before the first identity lookup");
+
+    // First call: no index -> scan fallback -> repair-walk backfills the sidecar.
+    let (out1, code1) = run_code_in(&dir, &["campaign", "validate", "identity-ok.campaign.json"]);
+    assert_eq!(code1, Some(0), "stdout/stderr: {out1}");
+    assert!(out1.contains(
+        "campaign document valid (referential): all references resolve, axes are in the param space"
+    ));
+    assert!(sidecar.is_file(), "the scan fallback must backfill the sidecar index");
+
+    // Second call: the sidecar now exists -> a verified index hit. Same answer.
+    let (out2, code2) = run_code_in(&dir, &["campaign", "validate", "identity-ok.campaign.json"]);
+    assert_eq!(code2, Some(0), "stdout/stderr: {out2}");
+    assert_eq!(out1, out2, "index-hit resolution must answer identically to the scan fallback");
+}
+
+/// Property (#191, negative twin): an `identity_id` ref that matches no
+/// stored blueprint is reported through the SAME `ref_fault_prose`
+/// `IdentityUnmatched` mapping as `content_id`'s `StrategyNotFound` — pinned
+/// as a unit-tested Debug-free string (`ref_fault_prose_is_debug_free`) but
+/// never before driven through the actual `campaign validate` CLI seam,
+/// so a wiring break between the fault type and its CLI dispatch would not
+/// have failed any existing test.
+#[test]
+fn campaign_validate_reports_unresolved_identity_ref_end_to_end() {
+    // Referential tier only runs inside a project (`campaign_validate_outside_project_skips_referential_tier`);
+    // a plain `temp_cwd` would silently pass at the intrinsic tier instead.
+    let (dir, _fixture) = fresh_project();
+    let runs_dir = dir.join("runs");
+    std::fs::remove_dir_all(&runs_dir).ok();
+    let _cleanup = ScratchGuard(vec![
+        ScratchPath::Dir(runs_dir),
+        ScratchPath::File(dir.join("identity-miss.campaign.json")),
+    ]);
+    let campaign = r#"{
+  "format_version": 1,
+  "kind": "campaign",
+  "name": "identity-ref-miss",
+  "data": { "instruments": ["GER40"], "windows": [ { "from_ms": 1, "to_ms": 2 } ] },
+  "strategies": [ { "ref": { "identity_id": "0000000000000000000000000000000000000000000000000000000000000000" },
+                    "axes": { "fast": { "kind": "I64", "values": [8] } } } ],
+  "process": { "ref": { "content_id": "4e2d" } },
+  "seed": 1,
+  "presentation": { "persist_taps": [], "emit": ["family_table"] }
+}"#;
+    write_doc(&dir, "identity-miss.campaign.json", campaign);
+    let (out, code) = run_code_in(&dir, &["campaign", "validate", "identity-miss.campaign.json"]);
+    assert_eq!(code, Some(1), "stdout/stderr: {out}");
+    assert!(out.contains("campaign references do not resolve:"), "stdout/stderr: {out}");
+    assert!(
+        out.contains("matches no stored blueprint"),
+        "the IdentityUnmatched fault prose must appear: {out}"
+    );
+}
+
 const CAMPAIGN_DOC: &str = r#"{
  "format_version": 1,
  "kind": "campaign",