Brummel
43b1b6fc25
The semantic index was ~90 896 per-entry files (index/embeddings/<safe-model>/
<sha256>.f32), a deployment liability: the file count — not the 356 MB — is
hostile to container layers, object storage, and Git-LFS, and Pipeline::load
opened every one individually. This packs a complete per-file store into one
self-describing matrix file loaded in a single read.
New src/packed.rs owns the format: 4-byte magic "AIPK", u32 LE format_version,
u32 LE header length, a JSON PackedHeader {n_rows, dim, embed_model,
corpus_sha256, corpus_name}, pad to a 4-byte boundary, then row-major
little-endian f32 payload in corpus order (row i = corpus entry i). write_packed
is atomic (temp+rename); read_packed rejects a wrong magic, a newer-than-
supported version, a malformed header, or a payload length inconsistent with
n_rows×dim. corpus_sha256 hashes the raw corpus file bytes.
Pipeline::load now calls a free fn load_packed_or_store(cfg, entries) that
prefers the packed file and falls back to today's all-or-nothing per-file store,
reporting which path it took via a new IndexStatus {Ok, Absent, Mismatch}
carried into Diagnostics. `alpha-id index --pack` writes the file from a
complete store (refusing an incomplete one with exit 2), recording corpus_sha256
+ embed_model in the header as the artifact-binding data.
Iteration boundary (deliberate): this is iteration 1 of the packed-versioned-
bundle cycle. load_packed_or_store validates the packed file STRUCTURALLY only
(magic/version/payload length via read_packed, plus n_rows == entries.len());
it does NOT yet compare the header's corpus_sha256/embed_model against the live
config, and IndexStatus::Mismatch fires only on structural corruption. The
load-time mismatch warning, the index= token on the diagnostics line, and the
Hybrid-degrade-names-reason wiring are iteration 2 (issue #4), which consumes
the binding data this commit writes. The per-file store is untouched — it stays
the build/resume path; the packed file is an additive serve-path artifact.
mmap was considered and rejected: the brute-force full-scan cosine search
touches every row per query, so memory-mapping yields no runtime benefit over a
single read and would add an unsafe dependency (spec § Out of scope).
Verification: cargo test green (61 tests; 9 new — 3 format round-trip/hash/bad-
magic, 4 load-path incl. packed-beats-store and structural n_rows mismatch, 2
--pack CLI incl. exit-2 refusal). An end-to-end seam test writes the packed
file via the CLI, removes the per-file store, and asserts the pipeline builds
its index from the packed file alone as IndexStatus::Ok (#3 acceptance b).
Spec: docs/specs/2026-05-31-packed-versioned-index-bundle.md
Plan: docs/plans/2026-06-01-packed-versioned-bundle-iter1.md
closes #3
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
123 lines
5.0 KiB
Rust
123 lines
5.0 KiB
Rust
use alpha_id::model::{AlphaIdEntry, Config, IndexStatus};
|
|
use alpha_id::pipeline::load_packed_or_store;
|
|
use std::path::Path;
|
|
use std::process::Command;
|
|
|
|
// Write a minimal config TOML pointing the corpus + index dir at temp paths.
|
|
// The `--pack` branch only reads `alpha_id_path` (corpus) and `index_dir`
|
|
// (store); `claml_path` etc. are present for parse but unused on this path.
|
|
fn write_config(dir: &Path, corpus: &Path, index_dir: &Path) -> std::path::PathBuf {
|
|
let toml = format!(
|
|
r#"ionos_base_url = "http://127.0.0.1:1"
|
|
token_path = "/nonexistent"
|
|
embed_model = "test/model"
|
|
rerank_model = "test/rerank"
|
|
alpha_id_path = "{corpus}"
|
|
claml_path = "/nonexistent.xml"
|
|
index_dir = "{index}"
|
|
pool_size = 10
|
|
top_k = 10
|
|
"#,
|
|
corpus = corpus.to_str().unwrap(),
|
|
index = index_dir.to_str().unwrap(),
|
|
);
|
|
let p = dir.join("config.toml");
|
|
std::fs::write(&p, toml).unwrap();
|
|
p
|
|
}
|
|
|
|
fn corpus_line(text: &str) -> String {
|
|
// Alpha-ID format: valid|alpha_id|primary|star|addon|primary2|orpha|text
|
|
format!("1|A0001|A00|||||{text}")
|
|
}
|
|
|
|
#[test]
|
|
fn pack_refuses_incomplete_store_exit_2() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let corpus = dir.path().join("corpus.txt");
|
|
std::fs::write(&corpus, format!("{}\n{}\n", corpus_line("alpha"), corpus_line("beta"))).unwrap();
|
|
let index_dir = dir.path().join("index");
|
|
std::fs::create_dir_all(&index_dir).unwrap();
|
|
let config = write_config(dir.path(), &corpus, &index_dir);
|
|
|
|
let out = Command::new(env!("CARGO_BIN_EXE_alpha-id"))
|
|
.args(["--config", config.to_str().unwrap(), "index", "--pack"])
|
|
.output()
|
|
.unwrap();
|
|
assert_eq!(out.status.code(), Some(2), "incomplete store must exit 2");
|
|
let stderr = String::from_utf8_lossy(&out.stderr);
|
|
assert!(stderr.contains("store incomplete"), "stderr was: {stderr}");
|
|
}
|
|
|
|
#[test]
|
|
fn pack_writes_file_from_complete_store() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let corpus = dir.path().join("corpus.txt");
|
|
std::fs::write(&corpus, format!("{}\n{}\n", corpus_line("alpha"), corpus_line("beta"))).unwrap();
|
|
let index_dir = dir.path().join("index");
|
|
std::fs::create_dir_all(&index_dir).unwrap();
|
|
|
|
// Populate the per-file store for both entry texts so the pack succeeds.
|
|
let store = alpha_id::embed::EmbeddingStore::open(index_dir.to_str().unwrap(), "test/model").unwrap();
|
|
store.put("alpha", &[1.0f32, 0.0]).unwrap();
|
|
store.put("beta", &[0.0f32, 1.0]).unwrap();
|
|
|
|
let config = write_config(dir.path(), &corpus, &index_dir);
|
|
let out = Command::new(env!("CARGO_BIN_EXE_alpha-id"))
|
|
.args(["--config", config.to_str().unwrap(), "index", "--pack"])
|
|
.output()
|
|
.unwrap();
|
|
assert!(out.status.success(), "stderr: {}", String::from_utf8_lossy(&out.stderr));
|
|
let packed = index_dir.join("packed.test_model.bin");
|
|
assert!(packed.exists(), "packed file should be written at {}", packed.display());
|
|
}
|
|
|
|
fn entry(text: &str) -> AlphaIdEntry {
|
|
AlphaIdEntry {
|
|
alpha_id: "A0001".to_string(),
|
|
valid: true,
|
|
icd_primary: "A00".to_string(),
|
|
icd_star: String::new(),
|
|
icd_addon: String::new(),
|
|
icd_primary2: String::new(),
|
|
orpha: String::new(),
|
|
text: text.to_string(),
|
|
}
|
|
}
|
|
|
|
/// End-to-end seam: a packed file produced by the `index --pack` CLI loads back
|
|
/// through the production `load_packed_or_store` path as `IndexStatus::Ok` with a
|
|
/// usable index, with no per-file store consulted. This pins the cross-boundary
|
|
/// round trip (CLI writer ↔ pipeline reader) that the two halves' unit tests
|
|
/// each cover only one side of.
|
|
#[test]
|
|
fn cli_packed_file_loads_back_as_ok_index() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let corpus = dir.path().join("corpus.txt");
|
|
std::fs::write(&corpus, format!("{}\n{}\n", corpus_line("alpha"), corpus_line("beta"))).unwrap();
|
|
let index_dir = dir.path().join("index");
|
|
std::fs::create_dir_all(&index_dir).unwrap();
|
|
|
|
let store = alpha_id::embed::EmbeddingStore::open(index_dir.to_str().unwrap(), "test/model").unwrap();
|
|
store.put("alpha", &[1.0f32, 0.0]).unwrap();
|
|
store.put("beta", &[0.0f32, 1.0]).unwrap();
|
|
|
|
let config = write_config(dir.path(), &corpus, &index_dir);
|
|
let out = Command::new(env!("CARGO_BIN_EXE_alpha-id"))
|
|
.args(["--config", config.to_str().unwrap(), "index", "--pack"])
|
|
.output()
|
|
.unwrap();
|
|
assert!(out.status.success(), "stderr: {}", String::from_utf8_lossy(&out.stderr));
|
|
|
|
// Load the CLI-written packed file through the production path. The store
|
|
// directory is removed first to prove the index comes from the packed file
|
|
// alone, not the per-file store.
|
|
std::fs::remove_dir_all(index_dir.join("embeddings")).unwrap();
|
|
let mut cfg = Config::load(config.to_str().unwrap()).unwrap();
|
|
cfg.index_dir = index_dir.to_str().unwrap().to_string();
|
|
let entries = vec![entry("alpha"), entry("beta")];
|
|
let (vector, status) = load_packed_or_store(&cfg, &entries);
|
|
assert!(vector.is_some(), "packed file written by the CLI must yield an index");
|
|
assert_eq!(status, IndexStatus::Ok);
|
|
}
|