From 6647a66bd08ec6d503d5a18540f85de36e4fcef3 Mon Sep 17 00:00:00 2001 From: Brummel Date: Mon, 18 May 2026 17:04:34 +0200 Subject: [PATCH] feat: Mode C pipeline end-to-end (lexical + tags + fusion) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement Pipeline::load/suggest for Mode C: lexical search per segment, cross-segment dedup via fusion, tag derivation, filter application. Add meta_lookup fallback (exact → strip trailing 0 → 3-char root) to bridge the ICD code granularity gap between the corpus (E11.90-style 6-char codes) and ClaML (which only stores E11, E11.0-style entries). --- src/pipeline.rs | 129 ++++++++++++++++++++++++++++++++- tests/pipeline_mode_c_tests.rs | 27 +++++++ 2 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 tests/pipeline_mode_c_tests.rs diff --git a/src/pipeline.rs b/src/pipeline.rs index 4b0fd9d..ae9c1b7 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -1 +1,128 @@ -// implemented in a later task +use crate::claml; +use crate::corpus::{self}; +use crate::fusion::cross_segment_dedupe; +use crate::lexical::LexicalIndex; +use crate::model::*; +use crate::segment::split_segments; +use crate::tags::tags_for; +use std::collections::HashMap; +use std::time::Instant; + +pub struct Pipeline { + pub entries: Vec, + pub meta: HashMap, + pub lexical: LexicalIndex, + pub pool_size: usize, +} + +impl Pipeline { + pub fn load(cfg: &Config) -> Result { + let entries = corpus::load(&cfg.alpha_id_path)?; + let meta = claml::load(&cfg.claml_path)?; + let lexical = LexicalIndex::build_in_ram(&entries)?; + Ok(Self { entries, meta, lexical, pool_size: cfg.pool_size }) + } + + /// Look up ICD metadata with fallback: exact code → strip trailing '0' after + /// the decimal (e.g. "E11.90" → "E11.9") → 3-char root (e.g. "E11"). + /// The corpus uses 5- and 6-char codes; ClaML uses 3-, 5-, and 6-char codes. + fn meta_lookup<'a>(&'a self, code: &str) -> Option<&'a IcdMeta> { + if let Some(m) = self.meta.get(code) { + return Some(m); + } + // Try stripping a trailing '0' from the last position after the dot + // e.g. "E11.90" → "E11.9", "I10.90" → "I10.9", "G40.00" → "G40.0" + if code.len() >= 5 { + if let Some(dot_pos) = code.find('.') { + let after_dot = &code[dot_pos + 1..]; + if after_dot.ends_with('0') && after_dot.len() > 1 { + let trimmed = &code[..code.len() - 1]; + if let Some(m) = self.meta.get(trimmed) { + return Some(m); + } + } + } + } + // Fallback to 3-char root (e.g. "E11") + if code.len() > 3 { + if let Some(m) = self.meta.get(&code[..3]) { + return Some(m); + } + } + None + } + + fn to_suggestion(&self, code: &str, phrase: &str, score: f32, + segs: Vec, alpha_valid: bool) -> Option { + let m = self.meta_lookup(code)?; + Some(Suggestion { + icd_code: code.to_string(), + description: m.description.clone(), + score, + matched_phrase: phrase.to_string(), + source_segments: segs, + tags: tags_for(m, alpha_valid), + }) + } + + pub fn suggest(&self, dictation: &str, mode: Mode, filter: &Filter, top_k: usize) + -> SuggestResult { + let t0 = Instant::now(); + let segments = split_segments(dictation); + let mut candidates: Vec = Vec::new(); + + for (si, seg) in segments.iter().enumerate() { + let lex = self.lexical.search(seg, self.pool_size).unwrap_or_default(); + for (entry_idx, score) in lex { + let e = &self.entries[entry_idx]; + if let Some(code) = corpus::primary_code(e) { + candidates.push(Candidate { + icd_code: code.to_string(), + alpha_text: e.text.clone(), + segment_idx: si, + lexical: Some(score), + semantic: None, + rerank: None, + }); + } + } + } + + let valid_by_code: HashMap = { + let mut m = HashMap::new(); + for c in &candidates { + m.entry(c.icd_code.clone()).or_insert(false); + } + for e in &self.entries { + if let Some(code) = corpus::primary_code(e) { + if let Some(v) = m.get_mut(code) { *v = *v || e.valid; } + } + } + m + }; + + let merged = cross_segment_dedupe(candidates); + let mut suggestions = Vec::new(); + for (d, score) in merged { + let alpha_valid = *valid_by_code.get(&d.icd_code).unwrap_or(&false); + if let Some(s) = self.to_suggestion(&d.icd_code, &d.best_phrase, score, + d.source_segments.clone(), alpha_valid) { + if filter.accepts(&s.tags) { + suggestions.push(s); + } + } + if suggestions.len() >= top_k { break; } + } + + SuggestResult { + suggestions, + diagnostics: Diagnostics { + mode: format!("{:?}", mode), + degraded: false, + segment_count: segments.len(), + ionos_calls: 0, + millis: t0.elapsed().as_millis(), + }, + } + } +} diff --git a/tests/pipeline_mode_c_tests.rs b/tests/pipeline_mode_c_tests.rs new file mode 100644 index 0000000..3fd3219 --- /dev/null +++ b/tests/pipeline_mode_c_tests.rs @@ -0,0 +1,27 @@ +use alpha_id::pipeline::Pipeline; +use alpha_id::model::{Config, Filter, Mode}; + +fn cfg() -> Config { + Config::load("config/default.toml").unwrap() +} + +#[test] +fn mode_c_end_to_end_real_data() { + let p = Pipeline::load(&cfg()).unwrap(); + let dictation = "Patient mit Diabetes mellitus Typ 2\n\nArterielle Hypertonie"; + let res = p.suggest(dictation, Mode::C, &Filter::default(), 10); + assert!(!res.suggestions.is_empty()); + assert!(res.suggestions.len() <= 10); + assert!(res.suggestions.iter().any(|s| s.icd_code.starts_with("E11"))); + assert!(res.suggestions.iter().any(|s| s.icd_code.starts_with("I10"))); + assert_eq!(res.diagnostics.segment_count, 2); + assert_eq!(res.diagnostics.mode, "C"); +} + +#[test] +fn billable_only_filter_excludes_three_digit_v_codes() { + let p = Pipeline::load(&cfg()).unwrap(); + let f = Filter { billable_only: true, valid_only: true, chapters: None, exclude_exotic: false }; + let res = p.suggest("Cholera", Mode::C, &f, 10); + assert!(res.suggestions.iter().all(|s| s.tags.billable && s.tags.valid)); +}