diff --git a/server/scripts/filter-hunspell-de.sh b/server/scripts/filter-hunspell-de.sh index 35ab0e3..b7e4c9c 100755 --- a/server/scripts/filter-hunspell-de.sh +++ b/server/scripts/filter-hunspell-de.sh @@ -8,8 +8,10 @@ # individual unknown words are kept. That matches our gazetteer's # token-level matching model (see gazetteer/mod.rs). # -# Dependencies: hunspell, hunspell-de-de (Debian: apt install hunspell -# hunspell-de-de). +# Dependencies: hunspell + German dictionary. Package names vary: +# Arch/CachyOS: pacman -S hunspell hunspell-de +# Debian/Ubuntu: apt install hunspell hunspell-de-de +# Fedora: dnf install hunspell hunspell-de # # Usage: ./filter-hunspell-de.sh @@ -23,20 +25,28 @@ fi INPUT="$1" OUTPUT="$2" +# Minimum entry length. Must match MIN_TOKEN_LEN in src/gazetteer/mod.rs. +# 5 is the current value — appropriate for curated medication lists where +# the 4-char eponym-vs-German-word collision class is not present. +MIN_LEN=5 + if ! command -v hunspell >/dev/null 2>&1; then - echo "Error: hunspell is not installed." >&2 - echo " Debian/Ubuntu: sudo apt install hunspell hunspell-de-de" >&2 + echo "Error: hunspell is not installed. Install it via your package" >&2 + echo " manager (e.g. pacman -S hunspell, apt install hunspell)." >&2 exit 1 fi # Sanity-check: is the de_DE dictionary available? if ! echo "Haus" | hunspell -d de_DE -a >/dev/null 2>&1; then - echo "Error: hunspell de_DE dictionary not available." >&2 - echo " Debian/Ubuntu: sudo apt install hunspell-de-de" >&2 + echo "Error: hunspell de_DE dictionary not available. Install:" >&2 + echo " Arch/CachyOS: pacman -S hunspell-de" >&2 + echo " Debian/Ubuntu: apt install hunspell-de-de" >&2 + echo " Fedora: dnf install hunspell-de" >&2 exit 1 fi # `hunspell -l` prints every unknown word from stdin, one per line. -# Pipe through sort -u to drop duplicates (multi-word titles can share -# tokens) and produce a stable output order. -hunspell -d de_DE -l < "$INPUT" | sort -u > "$OUTPUT" +# Drop words shorter than MIN_LEN, then sort -u for dedup + stable order. +hunspell -d de_DE -l < "$INPUT" \ + | awk -v min="$MIN_LEN" 'length($0) >= min' \ + | sort -u > "$OUTPUT" diff --git a/server/src/gazetteer/mod.rs b/server/src/gazetteer/mod.rs index c5192cd..03376f7 100644 --- a/server/src/gazetteer/mod.rs +++ b/server/src/gazetteer/mod.rs @@ -17,9 +17,12 @@ use std::path::Path; use strsim::damerau_levenshtein; -/// Tokens below this char count are never annotated. Short tokens produce -/// too many false positives (ACE, EKG, TAV). -const MIN_TOKEN_LEN: usize = 4; +/// Minimum char count for *both* gazetteer entries and transcript tokens. +/// Calibrated against curated medication lists: 5 keeps prototypical +/// 5-char brand names (Vomex, Tavor, Lasix, Nasic) while still blocking +/// the 4-char deutsche Alltagswort ↔ eponym collisions that unkurated +/// crawls produce (Bein↔Behn, nach↔Nuck). +const MIN_TOKEN_LEN: usize = 5; /// Maximum Damerau-Levenshtein distance for a candidate to qualify as /// a typo of a gazetteer entry. Distance 0 is excluded — the `exact` @@ -78,6 +81,11 @@ impl Gazetteer { } fn insert(&mut self, entry: &str) { + // Skip short entries — they cause noisy phonetic collisions with + // ordinary German words. Same threshold as token matching. + if entry.chars().count() < MIN_TOKEN_LEN { + return; + } let lower = entry.to_lowercase(); if !self.exact.insert(lower.into_boxed_str()) { // Duplicate (case-insensitive) — don't add to phonetic index either. @@ -127,20 +135,46 @@ impl Gazetteer { if self.exact.contains(lower.as_str()) { return None; } + + // Stage 1: phonetic-indexed lookup. Fast, filters out the vast + // majority of non-matches in O(1). Catches same-code neighbours + // like Klexane↔Clexane, Diacepam↔Diazepam. let code = koelner::encode(token); - if code.is_empty() { - return None; + if !code.is_empty() + && let Some(candidates) = self.by_phonetic.get(&code) + && let Some(hit) = best_candidate(&lower, candidates.iter().map(|c| c.as_ref())) + { + return Some(hit); } - let candidates = self.by_phonetic.get(&code)?; - candidates - .iter() - .map(|c| (damerau_levenshtein(&lower, &c.to_lowercase()), c)) - .filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE) - .min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.as_ref().cmp(b.1.as_ref()))) - .map(|(_, c)| c.as_ref()) + + // Stage 2: brute-force edit-distance over every entry. Catches + // ASR errors where Whisper substitutes phonetically-distinct + // consonants (Carvenilol↔Carvedilol: N↔D) or drops a letter + // that shifts the Kölner code (Acoxia↔Arcoxia: missing R). + // O(n) per unknown token; at n≈200 entries and ~50 tokens per + // transcript the absolute cost stays well under a millisecond. + best_candidate( + &lower, + self.by_phonetic.values().flatten().map(|c| c.as_ref()), + ) } } +/// Select the canonical entry from `candidates` that is closest to +/// `lower_token` by Damerau-Levenshtein distance, within the allowed +/// threshold. Distance 0 is rejected because that's already handled by +/// the `exact` short-circuit. Ties break alphabetically for determinism. +fn best_candidate<'a>( + lower_token: &str, + candidates: impl Iterator, +) -> Option<&'a str> { + candidates + .map(|c| (damerau_levenshtein(lower_token, &c.to_lowercase()), c)) + .filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE) + .min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(b.1))) + .map(|(_, c)| c) +} + /// A slice of the input text — either a word or a non-word run. enum Segment<'a> { Word(&'a str), @@ -191,28 +225,30 @@ mod tests { #[test] fn empty_gazetteer_is_pass_through() { let g = Gazetteer::empty(); - let input = "Patient nahm Womax gegen Übelkeit."; + let input = "Patient nahm Zerebrum-Medikation."; assert_eq!(g.annotate(input), input); } - /// Core regression: the Womax/Vomex use case. A token that's a - /// phonetic neighbor at edit distance ≤ 2 must get a hint appended. + /// Core regression: a token that's a phonetic neighbor at edit + /// distance ≤ 2 gets a hint appended. Cerebrum and Zerebrum both + /// encode to Kölner `87176` and differ by one char — a plausible + /// Whisper-style transcription slip. #[test] fn annotates_typo_for_phonetic_neighbor() { - let g = from_entries(&["Vomex"]); - let out = g.annotate("Patient nahm Womax gegen Übelkeit."); + let g = from_entries(&["Cerebrum"]); + let out = g.annotate("Patient mit Blutung im Zerebrum."); assert!( - out.contains("Womax [?Vomex]"), + out.contains("Zerebrum [?Cerebrum]"), "expected annotation, got: {out:?}" ); } #[test] fn noop_on_exact_match() { - let g = from_entries(&["Vomex"]); + let g = from_entries(&["Cerebrum"]); assert_eq!( - g.annotate("Patient nahm Vomex."), - "Patient nahm Vomex.", + g.annotate("Patient mit Blutung im Cerebrum."), + "Patient mit Blutung im Cerebrum.", ); } @@ -220,45 +256,76 @@ mod tests { /// the LLM with a hint for a token that's only case-different. #[test] fn noop_on_case_only_difference() { - let g = from_entries(&["Vomex"]); + let g = from_entries(&["Cerebrum"]); assert_eq!( - g.annotate("Patient nahm vomex."), - "Patient nahm vomex.", + g.annotate("Patient mit Blutung im cerebrum."), + "Patient mit Blutung im cerebrum.", ); } #[test] fn noop_on_unrelated_token() { - let g = from_entries(&["Vomex"]); + let g = from_entries(&["Cerebrum"]); let input = "Patient hat Husten und Fieber."; assert_eq!(g.annotate(input), input); } - /// Tokens shorter than MIN_TOKEN_LEN are never annotated, even if - /// the gazetteer contains a matching short entry. + /// Gazetteer entries shorter than MIN_TOKEN_LEN are rejected at load + /// time. This prevents 4-char eponyms (Nuck, Behn, Bell, …) from + /// generating noisy false-positive matches against everyday German + /// words — the empirical reason MIN_TOKEN_LEN exists. #[test] - fn noop_on_short_token() { - let g = from_entries(&["ACE"]); - let input = "ACE und AGE und APE"; + fn rejects_gazetteer_entries_below_min_length() { + let g = from_entries(&["Nuck", "Bein", "Behn"]); + assert_eq!(g.len(), 0, "short entries must be dropped"); + // Input with a matching short word passes through untouched. + let input = "Patient klagt nach schwerem Heben."; assert_eq!(g.annotate(input), input); } + /// Edit-distance fallback: Whisper substitutes phonetically-distinct + /// consonants (here N↔D in Carvenilol↔Carvedilol). The Kölner codes + /// differ (N=6 vs D=2), so stage-1 misses — the brute-force stage + /// must catch the distance-1 match. + #[test] + fn edit_distance_fallback_catches_consonant_substitution() { + let g = from_entries(&["Carvedilol"]); + let out = g.annotate("Patient nimmt Carvenilol."); + assert!( + out.contains("Carvenilol [?Carvedilol]"), + "expected fallback annotation, got: {out:?}" + ); + } + + /// Edit-distance fallback: Whisper drops a consonant (leading B in + /// Berodual). Kölner codes are completely different — only the + /// linear edit-distance scan saves the match. + #[test] + fn edit_distance_fallback_catches_dropped_consonant() { + let g = from_entries(&["Berodual"]); + let out = g.annotate("Patient bekommt Erodual."); + assert!( + out.contains("Erodual [?Berodual]"), + "expected fallback annotation, got: {out:?}" + ); + } + /// With two phonetically-colliding candidates, the one with smaller - /// Damerau-Levenshtein distance wins. Womax↔Wamax is 1, - /// Womax↔Vomex is 2 → Wamax is chosen. + /// Damerau-Levenshtein distance wins. Zerebrum↔Cerebrum = 1, + /// Zerebrum↔Cerebrom = 2 → Cerebrum is chosen. #[test] fn picks_closest_candidate_by_edit_distance() { - let g = from_entries(&["Vomex", "Wamax"]); - let out = g.annotate("Patient nahm Womax."); + let g = from_entries(&["Cerebrum", "Cerebrom"]); + let out = g.annotate("Patient mit Blutung im Zerebrum."); assert!( - out.contains("Womax [?Wamax]"), - "expected Wamax as closer candidate, got: {out:?}" + out.contains("Zerebrum [?Cerebrum]"), + "expected Cerebrum as closer candidate, got: {out:?}" ); } #[test] fn preserves_whitespace_and_punctuation() { - let g = from_entries(&["Zebra"]); + let g = from_entries(&["Cerebrum"]); let input = "Line 1.\n\nLine 2: foo-bar; baz!"; assert_eq!(g.annotate(input), input); } @@ -266,7 +333,7 @@ mod tests { #[tokio::test] async fn load_dir_reads_all_txt_files() { let dir = tempdir().unwrap(); - tokio::fs::write(dir.path().join("meds.txt"), "Vomex\nAspirin\n") + tokio::fs::write(dir.path().join("meds.txt"), "Aspirin\nIbuprofen\n") .await .unwrap(); tokio::fs::write(dir.path().join("anatomy.txt"), "Coracoideus\n") @@ -281,7 +348,7 @@ mod tests { let dir = tempdir().unwrap(); tokio::fs::write( dir.path().join("v.txt"), - "# medications\nVomex\n\n \n# another\nAspirin\n", + "# medications\nAspirin\n\n \n# another\nIbuprofen\n", ) .await .unwrap(); @@ -292,10 +359,10 @@ mod tests { #[tokio::test] async fn load_dir_ignores_non_txt_files() { let dir = tempdir().unwrap(); - tokio::fs::write(dir.path().join("v.txt"), "Vomex\n") + tokio::fs::write(dir.path().join("v.txt"), "Aspirin\n") .await .unwrap(); - tokio::fs::write(dir.path().join("readme.md"), "Aspirin\n") + tokio::fs::write(dir.path().join("readme.md"), "Ibuprofen\n") .await .unwrap(); let g = Gazetteer::load_dir(dir.path()).await.unwrap(); diff --git a/server/tests/analyze_test.rs b/server/tests/analyze_test.rs index 4b03270..b5c6a06 100644 --- a/server/tests/analyze_test.rs +++ b/server/tests/analyze_test.rs @@ -360,6 +360,131 @@ async fn analyze_worker_writes_document_via_wiremock() { let _ = tokio::time::timeout(Duration::from_secs(2), handle).await; } +/// The analyze worker must route transcripts through the Gazetteer before +/// sending them to the LLM. "Zerebrum" shares a Kölner-Phonetik code with +/// the gazetteer entry "Cerebrum" (both encode to `87176`) and differs by +/// exactly one character — so the annotation `Zerebrum [?Cerebrum]` must +/// end up in the LLM request body. +#[tokio::test] +async fn analyze_worker_annotates_with_gazetteer() { + let tmp = unique_tmp("g"); + let case_dir = tmp.join("dr_a/22222222-2222-2222-2222-222222222222"); + std::fs::create_dir_all(&case_dir).unwrap(); + + let input = json!({ + "last_recording_mtime": "2026-04-15T10:00:00Z", + "recordings": [ + { "recorded_at": "2026-04-15T10:00:00Z", "text": "Patient mit Blutung im Zerebrum." } + ] + }); + std::fs::write( + case_dir.join("analysis_input.json"), + serde_json::to_vec_pretty(&input).unwrap(), + ) + .unwrap(); + + // Vocab directory with a single anatomy entry. + let vocab_dir = unique_tmp("vg"); + std::fs::create_dir_all(&vocab_dir).unwrap(); + std::fs::write(vocab_dir.join("anatomy.txt"), "Cerebrum\n").unwrap(); + let vocab = Arc::new( + doctate_server::gazetteer::Gazetteer::load_dir(&vocab_dir) + .await + .unwrap(), + ); + assert_eq!(vocab.len(), 1); + + // Mock LLM: responds 200 unconditionally. The assertion is on the + // *request body*, not the response. + let mock = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "choices": [{ + "message": { "content": "Patient mit Blutung im ==Cerebrum==." } + }] + }))) + .mount(&mock) + .await; + + let config = config_with_llm(tmp.clone(), mock.uri()); + let (tx, rx) = analyze::channel(); + let client = reqwest::Client::new(); + let busy = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let handle = tokio::spawn(analyze::worker::run(rx, config, client, busy, vocab)); + + tx.send(analyze::AnalyzeJob { + case_dir: case_dir.clone(), + }) + .unwrap(); + + let document_path = case_dir.join("document.md"); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !document_path.exists() { + if std::time::Instant::now() > deadline { + panic!("document.md not written within 5s"); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + + // Inspect what actually went to the LLM. + let requests = mock.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1, "expected exactly one LLM request"); + let body = String::from_utf8(requests[0].body.clone()).unwrap(); + assert!( + body.contains("Zerebrum [?Cerebrum]"), + "expected gazetteer annotation in LLM request body, got:\n{body}", + ); + + drop(tx); + let _ = tokio::time::timeout(Duration::from_secs(2), handle).await; +} + +/// Manual debug utility — not part of the default test set. +/// Usage: +/// DRY_CASE_DIR=/path/to/case cargo test --test analyze_test annotate_real_case_dry -- --ignored --nocapture +/// +/// Prints each transcript in a given case dir side-by-side with its +/// gazetteer-annotated version. Useful to eyeball false-positives / +/// false-negatives on real cases. +#[tokio::test] +#[ignore] +async fn annotate_real_case_dry() { + let vocab_dir = std::env::var("DRY_VOCAB_DIR") + .unwrap_or_else(|_| "/home/brummel/dev/doctate/server/vocab".into()); + let case_dir = std::env::var("DRY_CASE_DIR").unwrap_or_else(|_| { + "/home/brummel/dev/doctate/tmpdata/dr_mueller/e817d8b8-43eb-436a-8364-76b80a1e627b".into() + }); + + let vocab = doctate_server::gazetteer::Gazetteer::load_dir(Path::new(&vocab_dir)) + .await + .expect("load vocab"); + println!("Gazetteer entries: {}", vocab.len()); + + let mut names: Vec<_> = std::fs::read_dir(&case_dir) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| { + p.file_name() + .and_then(|n| n.to_str()) + .map(|s| s.ends_with(".transcript.txt")) + .unwrap_or(false) + }) + .collect(); + names.sort(); + + for path in names { + let text = std::fs::read_to_string(&path).unwrap(); + let annotated = vocab.annotate(&text); + println!("\n=== {} ===", path.file_name().unwrap().to_string_lossy()); + println!("--- original ---\n{text}"); + println!("--- annotated ---\n{annotated}"); + if annotated == text { + println!("[no annotations applied]"); + } + } +} + // --------------------------------------------------------------------- // Recovery // --------------------------------------------------------------------- diff --git a/server/vocab/anatomy.txt b/server/vocab/anatomy.txt index 3bbbd6e..0f784ea 100644 --- a/server/vocab/anatomy.txt +++ b/server/vocab/anatomy.txt @@ -1,2 +1,3 @@ -# Anatomical Latin/Greek terms. Populate via ./scripts/build-anatomy.sh. -# Blank — mechanism runs as no-op until content is built. +# Anatomical Latin/Greek terms — currently disabled. +# Scope narrowed to medications. If re-enabling, populate via +# ./scripts/build-anatomy.sh and expect eponym-beifang. diff --git a/server/vocab/medications.txt b/server/vocab/medications.txt index 8b879b5..d9174cd 100644 --- a/server/vocab/medications.txt +++ b/server/vocab/medications.txt @@ -1,2 +1,165 @@ -# Exotic medication brand names. Population script TBD. -# Candidate sources: Wikidata (P267 ATC-code joins), Rote Liste alternatives. +# Seed list of German medication brand names + commonly dictated +# active-ingredient names. Roughly aligned with the AOK Arzneiverordnungs- +# Report top-prescribed set. Curated, not crawled — new entries should +# be ≥5 chars (phonetic-collision safety) and phonetically distinctive. +# Maintain alphabetical order, one entry per line, no inline comments. +Adalat +Aldactone +Allopurinol +Amiodaron +Amlodipin +Amoxicillin +Arcoxia +Aspirin +Atacand +Atorvastatin +Azithromycin +Beloc +Benazepril +Berodual +Betahistin +Bisoprolol +Budesonid +Buscopan +Candesartan +Carvedilol +Cefuroxim +Celebrex +Cetirizin +Ciprofloxacin +Citalopram +Clarithromycin +Clexane +Clindamycin +Clopidogrel +Concor +Cortison +Dabigatran +Dexamethason +Diazepam +Diclofenac +Digoxin +Diltiazem +Dimenhydrinat +Dobendan +Dolormin +Doxycyclin +Duloxetin +Edoxaban +Eliquis +Enalapril +Erythromycin +Escitalopram +Esomeprazol +Euthyrox +Fluoxetin +Foradil +Forxiga +Furosemid +Gabapentin +Glucophage +Haldol +Heparin +Humira +Ibuprofen +Iberogast +Imodium +Insulin +Isoptin +Januvia +Jardiance +Ketamin +Keppra +Klacid +Lactulose +Lansoprazol +Lantus +Lasix +Laxoberal +Lisinopril +Lixiana +Lorazepam +Lorzaar +Lovastatin +Macrogol +Marcumar +Mesalazin +Metformin +Methotrexat +Metoprolol +Metronidazol +Mirtazapin +Morphium +Movicol +Mucofalk +Naproxen +Nasivin +Nebivolol +Nexium +Nitrolingual +Norvasc +Novalgin +Novorapid +Olanzapin +Olynth +Omeprazol +Otriven +Oxazepam +Oxycodon +Ozempic +Pantoprazol +Paracetamol +Penicillin +Perenterol +Phenprocoumon +Pradaxa +Pravastatin +Prednisolon +Prednison +Pregabalin +Pulmicort +Quetiapin +Ramipril +Ranitidin +Rhinospray +Risperidon +Rivaroxaban +Rosuvastatin +Salbutamol +Sertralin +Sildenafil +Simvastatin +Sinupret +Sirdalud +Spiriva +Spironolacton +Stilnox +Sumatriptan +Symbicort +Tadalafil +Tavor +Telmisartan +Temesta +Thyroxin +Timolol +Torasemid +Tramadol +Tranxilium +Trulicity +Ultracain +Valsartan +Venlafaxin +Ventolin +Verapamil +Viagra +Viani +Victoza +Voltaren +Voluven +Vomex +Warfarin +Xarelto +Ximovan +Zolpidem +Zopiclon +Zyprexa