feat: cost-guarded resumable index build (sample smoke + full)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+41
-2
@@ -52,9 +52,48 @@ fn main() {
|
||||
let cfg = Config::load(&cli.config).expect("config");
|
||||
match cli.cmd {
|
||||
Cmd::Index { sample, full, confirm } => {
|
||||
use alpha_id::embed::{EmbeddingStore, build_corpus_embeddings, estimate};
|
||||
use alpha_id::ionos::IonosClient;
|
||||
let p = Pipeline::load(&cfg).expect("load");
|
||||
eprintln!("Loaded {} entries, {} ICD metas. (sample={:?} full={} confirm={})",
|
||||
p.entries.len(), p.meta.len(), sample, full, confirm);
|
||||
let texts: Vec<String> = p.entries.iter().map(|e| e.text.clone()).collect();
|
||||
let client = IonosClient::new(&cfg.ionos_base_url, &cfg.token_path)
|
||||
.expect("ionos client (token?)");
|
||||
let store = EmbeddingStore::open(&cfg.index_dir, &cfg.embed_model).expect("store");
|
||||
|
||||
let subset: Vec<String> = match sample {
|
||||
Some(n) => texts.iter().take(n).cloned().collect(),
|
||||
None => texts.clone(),
|
||||
};
|
||||
|
||||
if sample.is_some() {
|
||||
let est = estimate(&subset);
|
||||
eprintln!("SMOKE: embedding {} sample texts ({} chars)…", est.items, est.chars);
|
||||
let n = build_corpus_embeddings(&client, &store, &cfg.embed_model, &subset, 32)
|
||||
.expect("sample embed failed");
|
||||
let dim = store.get(&subset[0]).map(|v| v.len()).unwrap_or(0);
|
||||
eprintln!("SMOKE OK: embedded {n}, vector dim = {dim}. \
|
||||
Inspect, then run `index --full --confirm`.");
|
||||
return;
|
||||
}
|
||||
|
||||
if full {
|
||||
let pending: Vec<String> = texts.iter().filter(|t| !store.has(t)).cloned().collect();
|
||||
let est = estimate(&pending);
|
||||
eprintln!("FULL RUN: {} of {} texts not yet embedded ({} chars). \
|
||||
This calls IONOS and costs money.", est.items, texts.len(), est.chars);
|
||||
if !confirm {
|
||||
eprintln!("Refusing without --confirm. Re-run: \
|
||||
`alpha-id index --full --confirm`");
|
||||
std::process::exit(2);
|
||||
}
|
||||
let n = build_corpus_embeddings(&client, &store, &cfg.embed_model, &texts, 32)
|
||||
.expect("full embed failed");
|
||||
eprintln!("FULL RUN done: {n} newly embedded, {} total cached.",
|
||||
texts.iter().filter(|t| store.has(t)).count());
|
||||
return;
|
||||
}
|
||||
|
||||
eprintln!("Specify --sample N (smoke test first) or --full --confirm.");
|
||||
}
|
||||
Cmd::Suggest { input, mode, top, billable_only, valid_only, exclude_exotic, chapters, json } => {
|
||||
let p = Pipeline::load(&cfg).expect("load");
|
||||
|
||||
Reference in New Issue
Block a user