feat: packed single-file embedding index (closes #3)
The semantic index was ~90 896 per-entry files (index/embeddings/<safe-model>/
<sha256>.f32), a deployment liability: the file count — not the 356 MB — is
hostile to container layers, object storage, and Git-LFS, and Pipeline::load
opened every one individually. This packs a complete per-file store into one
self-describing matrix file loaded in a single read.
New src/packed.rs owns the format: 4-byte magic "AIPK", u32 LE format_version,
u32 LE header length, a JSON PackedHeader {n_rows, dim, embed_model,
corpus_sha256, corpus_name}, pad to a 4-byte boundary, then row-major
little-endian f32 payload in corpus order (row i = corpus entry i). write_packed
is atomic (temp+rename); read_packed rejects a wrong magic, a newer-than-
supported version, a malformed header, or a payload length inconsistent with
n_rows×dim. corpus_sha256 hashes the raw corpus file bytes.
Pipeline::load now calls a free fn load_packed_or_store(cfg, entries) that
prefers the packed file and falls back to today's all-or-nothing per-file store,
reporting which path it took via a new IndexStatus {Ok, Absent, Mismatch}
carried into Diagnostics. `alpha-id index --pack` writes the file from a
complete store (refusing an incomplete one with exit 2), recording corpus_sha256
+ embed_model in the header as the artifact-binding data.
Iteration boundary (deliberate): this is iteration 1 of the packed-versioned-
bundle cycle. load_packed_or_store validates the packed file STRUCTURALLY only
(magic/version/payload length via read_packed, plus n_rows == entries.len());
it does NOT yet compare the header's corpus_sha256/embed_model against the live
config, and IndexStatus::Mismatch fires only on structural corruption. The
load-time mismatch warning, the index= token on the diagnostics line, and the
Hybrid-degrade-names-reason wiring are iteration 2 (issue #4), which consumes
the binding data this commit writes. The per-file store is untouched — it stays
the build/resume path; the packed file is an additive serve-path artifact.
mmap was considered and rejected: the brute-force full-scan cosine search
touches every row per query, so memory-mapping yields no runtime benefit over a
single read and would add an unsafe dependency (spec § Out of scope).
Verification: cargo test green (61 tests; 9 new — 3 format round-trip/hash/bad-
magic, 4 load-path incl. packed-beats-store and structural n_rows mismatch, 2
--pack CLI incl. exit-2 refusal). An end-to-end seam test writes the packed
file via the CLI, removes the per-file store, and asserts the pipeline builds
its index from the packed file alone as IndexStatus::Ok (#3 acceptance b).
Spec: docs/specs/2026-05-31-packed-versioned-index-bundle.md
Plan: docs/plans/2026-06-01-packed-versioned-bundle-iter1.md
closes #3
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+35
-2
@@ -15,7 +15,7 @@ struct Cli {
|
||||
#[derive(Subcommand)]
|
||||
enum Cmd {
|
||||
/// Build indexes (lexical now; embeddings added in Phase A)
|
||||
Index { #[arg(long)] sample: Option<usize>, #[arg(long)] full: bool, #[arg(long)] confirm: bool },
|
||||
Index { #[arg(long)] sample: Option<usize>, #[arg(long)] full: bool, #[arg(long)] confirm: bool, #[arg(long)] pack: bool },
|
||||
/// Suggest ICD codes for a dictation (FILE or - for stdin)
|
||||
Suggest {
|
||||
input: String,
|
||||
@@ -80,9 +80,42 @@ fn main() {
|
||||
let cli = Cli::parse();
|
||||
let cfg = Config::load(&cli.config).expect("config");
|
||||
match cli.cmd {
|
||||
Cmd::Index { sample, full, confirm } => {
|
||||
Cmd::Index { sample, full, confirm, pack } => {
|
||||
use alpha_id::embed::{EmbeddingStore, build_corpus_embeddings, estimate};
|
||||
use alpha_id::ionos::IonosClient;
|
||||
|
||||
if pack {
|
||||
use alpha_id::packed::{corpus_sha256, packed_path, write_packed, PackedHeader};
|
||||
let entries = alpha_id::corpus::load(&cfg.alpha_id_path).expect("corpus");
|
||||
let store = EmbeddingStore::open(&cfg.index_dir, &cfg.embed_model).expect("store");
|
||||
let cached = entries.iter().filter(|e| store.has(&e.text)).count();
|
||||
if cached != entries.len() {
|
||||
eprintln!(
|
||||
"error: store incomplete: {} of {} entries embedded; \
|
||||
run `index --full --confirm` first",
|
||||
cached, entries.len()
|
||||
);
|
||||
std::process::exit(2);
|
||||
}
|
||||
let rows: Vec<Vec<f32>> =
|
||||
entries.iter().map(|e| store.get(&e.text).expect("cached vector")).collect();
|
||||
let dim = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
|
||||
let corpus_name = std::path::Path::new(&cfg.alpha_id_path)
|
||||
.file_name().and_then(|n| n.to_str())
|
||||
.unwrap_or(&cfg.alpha_id_path).to_string();
|
||||
let header = PackedHeader {
|
||||
n_rows: entries.len() as u32,
|
||||
dim,
|
||||
embed_model: cfg.embed_model.clone(),
|
||||
corpus_sha256: corpus_sha256(&cfg.alpha_id_path).expect("corpus hash"),
|
||||
corpus_name,
|
||||
};
|
||||
let path = packed_path(&cfg.index_dir, &cfg.embed_model);
|
||||
write_packed(&path, &header, &rows).expect("write packed");
|
||||
eprintln!("packed {} rows × {} dims → {}", header.n_rows, header.dim, path.display());
|
||||
return;
|
||||
}
|
||||
|
||||
let p = Pipeline::load(&cfg).expect("load");
|
||||
let texts: Vec<String> = p.entries.iter().map(|e| e.text.clone()).collect();
|
||||
let client = IonosClient::new(&cfg.ionos_base_url, &cfg.token_path)
|
||||
|
||||
Reference in New Issue
Block a user