feat: enforce + surface packed index binding (closes #4)
Iteration 1 shipped the packed format with the corpus_sha256 + embed_model
manifest in its header but only validated structure at load. A corpus release
or an embed_model change could still ship an index that loads "successfully"
yet belongs to a different corpus/model, and the only signal was the anonymous
degraded:true — indistinguishable from a transient IONOS outage. This closes
that gap.
load_packed_or_store now, after the structural checks pass, compares the
header's embed_model against cfg.embed_model and its corpus_sha256 against a
fresh hash of cfg.alpha_id_path. On either disagreement it returns
IndexStatus::Mismatch("embed model" | "corpus sha256") with no vector index,
so Mode Hybrid degrades to Lexical rather than serving a matrix that does not
belong to the deployed corpus/model. The check is ordered model-then-corpus so
a model mismatch never reads the corpus file.
The mismatch is now observable two ways, not just via degraded:
- Pipeline::load emits a prominent stderr warning naming the packed path and
the reason when the index is a Mismatch.
- IndexStatus gains a Display (ok / absent / mismatch: <reason>) and the human
diagnostics line carries an index=<status> token, e.g.
"[3 segments, mode Lexical, 12 ms, degraded=false, index=mismatch: corpus sha256]".
The index_status also rides in the JSON diagnostics (it was already a
serialized Diagnostics field from iteration 1). index_status is orthogonal to
degraded: a transient IONOS degrade leaves index_status=Ok, a structural
binding problem sets Mismatch.
Tests (67 green, +6 net): hermetic unit tests for the model- and
corpus-hash-mismatch rejections and a matching-pair acceptance, a Display
render test, a lib test that a Mode Hybrid suggest under a mismatch is
degraded with index_status=Mismatch (degrading before any IONOS post, so no
network), an end-to-end CLI test asserting the load warning and the
index=mismatch token reach stderr, and the existing transient-degrade test
extended with the index_status=Ok foil. The iteration-1
packed_beats_store_builds_without_store test was repaired (hermetic temp corpus
+ a real header hash) since the new semantic enforcement necessarily rejects
its previous structural-only-era placeholder hash; its intent (packed beats
store) is unchanged.
No IONOS calls are made by any test. The per-file store, embedding build, and
the VectorIndex are untouched. mmap remains rejected (spec § Out of scope).
Spec: docs/specs/2026-05-31-packed-versioned-index-bundle.md
Plan: docs/plans/2026-06-01-packed-versioned-bundle-iter2.md
closes #4
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
use alpha_id::model::{AlphaIdEntry, Config, IndexStatus};
|
||||
use alpha_id::packed::{packed_path, write_packed, PackedHeader};
|
||||
use alpha_id::packed::corpus_sha256;
|
||||
use alpha_id::pipeline::load_packed_or_store;
|
||||
|
||||
fn entry(text: &str) -> AlphaIdEntry {
|
||||
@@ -36,14 +37,19 @@ fn index_status_serializes_externally_tagged() {
|
||||
#[test]
|
||||
fn packed_beats_store_builds_without_store() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let cfg = cfg_in(dir.path());
|
||||
let mut cfg = cfg_in(dir.path());
|
||||
// Hermetic corpus so the header's corpus_sha256 can match the live config
|
||||
// under iteration-2 semantic enforcement.
|
||||
let corpus = dir.path().join("corpus.txt");
|
||||
std::fs::write(&corpus, b"some corpus bytes").unwrap();
|
||||
cfg.alpha_id_path = corpus.to_str().unwrap().to_string();
|
||||
let entries = vec![entry("a"), entry("b")];
|
||||
let rows = vec![vec![1.0f32, 0.0], vec![0.0f32, 1.0]];
|
||||
let h = PackedHeader {
|
||||
n_rows: 2,
|
||||
dim: 2,
|
||||
embed_model: cfg.embed_model.clone(),
|
||||
corpus_sha256: "irrelevant-in-iter1".to_string(),
|
||||
corpus_sha256: corpus_sha256(&cfg.alpha_id_path).unwrap(),
|
||||
corpus_name: "c.txt".to_string(),
|
||||
};
|
||||
write_packed(&packed_path(&cfg.index_dir, &cfg.embed_model), &h, &rows).unwrap();
|
||||
@@ -71,3 +77,74 @@ fn n_rows_mismatch_yields_mismatch_none() {
|
||||
assert!(vector.is_none());
|
||||
assert!(matches!(status, IndexStatus::Mismatch(_)));
|
||||
}
|
||||
|
||||
fn header_for(cfg: &Config, n_rows: u32, model: &str, corpus_hash: &str) -> PackedHeader {
|
||||
let _ = cfg;
|
||||
PackedHeader {
|
||||
n_rows,
|
||||
dim: 2,
|
||||
embed_model: model.to_string(),
|
||||
corpus_sha256: corpus_hash.to_string(),
|
||||
corpus_name: "c.txt".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_status_display_renders_spec_tokens() {
|
||||
assert_eq!(format!("{}", IndexStatus::Ok), "ok");
|
||||
assert_eq!(format!("{}", IndexStatus::Absent), "absent");
|
||||
assert_eq!(
|
||||
format!("{}", IndexStatus::Mismatch("corpus sha256".to_string())),
|
||||
"mismatch: corpus sha256"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_mismatch_yields_mismatch_none() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let cfg = cfg_in(dir.path()); // cfg.embed_model == "test/model"
|
||||
let entries = vec![entry("a"), entry("b")];
|
||||
let rows = vec![vec![1.0f32, 0.0], vec![0.0f32, 1.0]];
|
||||
// n_rows matches (2) so the structural check passes; the header declares a
|
||||
// different model, so the semantic check must reject it.
|
||||
let h = header_for(&cfg, 2, "other/model", "irrelevant");
|
||||
write_packed(&packed_path(&cfg.index_dir, &cfg.embed_model), &h, &rows).unwrap();
|
||||
let (vector, status) = load_packed_or_store(&cfg, &entries);
|
||||
assert!(vector.is_none());
|
||||
assert!(matches!(status, IndexStatus::Mismatch(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corpus_hash_mismatch_yields_mismatch_none() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut cfg = cfg_in(dir.path());
|
||||
// Point the corpus at a tiny temp file so the hash is hermetic.
|
||||
let corpus = dir.path().join("corpus.txt");
|
||||
std::fs::write(&corpus, b"some corpus bytes").unwrap();
|
||||
cfg.alpha_id_path = corpus.to_str().unwrap().to_string();
|
||||
let entries = vec![entry("a"), entry("b")];
|
||||
let rows = vec![vec![1.0f32, 0.0], vec![0.0f32, 1.0]];
|
||||
// Model matches, but the header's corpus hash does not match the file's.
|
||||
let h = header_for(&cfg, 2, &cfg.embed_model, "deadbeef-not-the-real-hash");
|
||||
write_packed(&packed_path(&cfg.index_dir, &cfg.embed_model), &h, &rows).unwrap();
|
||||
let (vector, status) = load_packed_or_store(&cfg, &entries);
|
||||
assert!(vector.is_none());
|
||||
assert!(matches!(status, IndexStatus::Mismatch(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matching_model_and_corpus_yields_ok() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut cfg = cfg_in(dir.path());
|
||||
let corpus = dir.path().join("corpus.txt");
|
||||
std::fs::write(&corpus, b"some corpus bytes").unwrap();
|
||||
cfg.alpha_id_path = corpus.to_str().unwrap().to_string();
|
||||
let entries = vec![entry("a"), entry("b")];
|
||||
let rows = vec![vec![1.0f32, 0.0], vec![0.0f32, 1.0]];
|
||||
let real_hash = corpus_sha256(&cfg.alpha_id_path).unwrap();
|
||||
let h = header_for(&cfg, 2, &cfg.embed_model, &real_hash);
|
||||
write_packed(&packed_path(&cfg.index_dir, &cfg.embed_model), &h, &rows).unwrap();
|
||||
let (vector, status) = load_packed_or_store(&cfg, &entries);
|
||||
assert!(vector.is_some());
|
||||
assert_eq!(status, IndexStatus::Ok);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user