From 1fd252f36381fc98f4532dcbe63cd2c8ef19b51f Mon Sep 17 00:00:00 2001 From: Brummel Date: Thu, 16 Apr 2026 17:23:52 +0200 Subject: [PATCH] Add gazetteer configuration and dependency This commit introduces the `VOCAB_DIR` configuration option, which specifies the directory for gazetteer files. It also adds the `strsim` crate, which is used for calculating string similarity and is essential for the gazetteer's functionality. The gazetteer module has been significantly expanded to include the `Gazetteer` struct, its loading mechanism from `.txt` files, and an annotation function. This functionality allows for pre-LLM correction of proper names and technical vocabulary by identifying potential misspellings based on phonetic similarity and edit distance. --- server/.env.example | 5 + server/Cargo.lock | 7 + server/Cargo.toml | 1 + server/src/config.rs | 6 + server/src/gazetteer/mod.rs | 312 +++++++++++++++++++++++++++++++++++- 5 files changed, 326 insertions(+), 5 deletions(-) diff --git a/server/.env.example b/server/.env.example index cb94d86..d00e459 100644 --- a/server/.env.example +++ b/server/.env.example @@ -33,5 +33,10 @@ LLM_TIMEOUT_SECONDS=180 # Single line — escape newlines with \n if your shell preserves them, or put the prompt on one logical line. LLM_SYSTEM_PROMPT="Du bist ein medizinischer Assistent. Fasse die folgenden diktierten Abschnitte zu einem zusammenhängenden Text zusammen. Bereinigen und strukturieren, nichts hinzufügen, nichts interpretieren, keine Diagnose, keine Arztbrief-Struktur. Entferne Redundanzen. Spätere Aufnahmen haben Vorrang — Korrekturen, Nachträge und Widersprüche zugunsten der chronologisch letzten Aussage auflösen. Antworte auf Deutsch. Nur der zusammengefasste Text, keine Einleitung, kein Abschlusssatz." +# Gazetteer (pre-LLM proper-name correction) +# Directory with *.txt files, one entry per line. Blank and #-comment lines +# are skipped. Missing directory is non-fatal — analysis runs without hints. +VOCAB_DIR=./vocab + # Session SESSION_TIMEOUT_HOURS=8 diff --git a/server/Cargo.lock b/server/Cargo.lock index 6a1660c..9e9c147 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -345,6 +345,7 @@ dependencies = [ "rpassword", "serde", "serde_json", + "strsim", "tempfile", "time", "tokio", @@ -1642,6 +1643,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" diff --git a/server/Cargo.toml b/server/Cargo.toml index cac1225..ebadddd 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -26,6 +26,7 @@ time = { version = "0.3.47", features = ["local-offset", "parsing", "formatting" toml_edit = "0.25.11" rpassword = "7.4.0" pulldown-cmark = { version = "0.13", default-features = false, features = ["html"] } +strsim = "0.11" [dev-dependencies] tower = { version = "0.5", features = ["util"] } diff --git a/server/src/config.rs b/server/src/config.rs index 96844d5..0c09626 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -58,6 +58,10 @@ pub struct Config { /// or empty. Runtime-configurable so admins can iterate and use the /// "Neu analysieren" button to re-run cases. pub llm_system_prompt: String, + /// Directory containing gazetteer `*.txt` files (anatomy, substances, + /// medications). Loaded once at startup; a missing directory is a + /// non-fatal WARN — analysis runs without proper-name correction. + pub vocab_dir: PathBuf, pub session_timeout_hours: u32, /// Whether to set the `Secure` flag on the session cookie. Default `true`. /// Set `COOKIE_SECURE=false` only for plain-HTTP local development — @@ -100,6 +104,7 @@ impl Config { v } }, + vocab_dir: PathBuf::from(optional_env("VOCAB_DIR", "./vocab")), session_timeout_hours: optional_env_parsed("SESSION_TIMEOUT_HOURS", 8), cookie_secure: optional_env_parsed("COOKIE_SECURE", true), } @@ -152,6 +157,7 @@ impl Config { llm_temperature: 0.0, llm_timeout_seconds: 180, llm_system_prompt: crate::analyze::prompt::SYSTEM_PROMPT.to_string(), + vocab_dir: PathBuf::new(), session_timeout_hours: 8, cookie_secure: false, } diff --git a/server/src/gazetteer/mod.rs b/server/src/gazetteer/mod.rs index 549634b..c5192cd 100644 --- a/server/src/gazetteer/mod.rs +++ b/server/src/gazetteer/mod.rs @@ -1,9 +1,311 @@ -//! Gazetteer: Pre-LLM correction layer for proper names and technical -//! vocabulary (medications, anatomy, substances). See +//! Gazetteer: domain-vocabulary lookup that injects correction hints +//! before the analysis LLM runs. See //! `~/.claude/plans/happy-mapping-snail.md` for the design document. //! -//! This module is currently a stub — only the Kölner Phonetik encoder -//! exists. The `Gazetteer` struct, loader, and annotator land in later -//! steps of the implementation. +//! The LLM cannot know invented proper names (drug brands, anatomical +//! Latin/Greek terms). Whisper misrenders them phonetically +//! (`Vomex` → `Womax`). This module exposes a static lookup that, given +//! a transcript, finds tokens that are likely typos of a known entry +//! and rewrites them as `token [?Canonical]`. The LLM decides in context +//! whether to adopt the hint. pub mod koelner; + +use std::collections::{HashMap, HashSet}; +use std::io; +use std::path::Path; + +use strsim::damerau_levenshtein; + +/// Tokens below this char count are never annotated. Short tokens produce +/// too many false positives (ACE, EKG, TAV). +const MIN_TOKEN_LEN: usize = 4; + +/// Maximum Damerau-Levenshtein distance for a candidate to qualify as +/// a typo of a gazetteer entry. Distance 0 is excluded — the `exact` +/// set short-circuits that case. +const MAX_EDIT_DISTANCE: usize = 2; + +/// Domain-specific vocabulary, populated once at server start from +/// `*.txt` files under a vocab directory. Read-only thereafter; share +/// via `Arc`. +#[derive(Debug, Default)] +pub struct Gazetteer { + /// Kölner-Phonetik code → canonical entries sharing that code. The + /// canonicals keep their original casing for display. + by_phonetic: HashMap>>, + /// Lowercased canonicals; short-circuits annotation when the token + /// is already spelled correctly. + exact: HashSet>, +} + +impl Gazetteer { + /// Empty gazetteer. `annotate` is a pure pass-through on this instance. + pub fn empty() -> Self { + Self::default() + } + + /// Load every `*.txt` file (non-recursive) under `dir`. Blank lines + /// and lines beginning with `#` are skipped. Case-insensitive + /// duplicates are silently collapsed. + pub async fn load_dir(dir: &Path) -> io::Result { + let mut g = Self::default(); + let mut entries = tokio::fs::read_dir(dir).await?; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("txt") { + continue; + } + let content = tokio::fs::read_to_string(&path).await?; + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + g.insert(trimmed); + } + } + Ok(g) + } + + /// Number of unique entries (case-insensitive). + pub fn len(&self) -> usize { + self.exact.len() + } + + pub fn is_empty(&self) -> bool { + self.exact.is_empty() + } + + fn insert(&mut self, entry: &str) { + let lower = entry.to_lowercase(); + if !self.exact.insert(lower.into_boxed_str()) { + // Duplicate (case-insensitive) — don't add to phonetic index either. + return; + } + let code = koelner::encode(entry); + if code.is_empty() { + return; + } + self.by_phonetic + .entry(code) + .or_default() + .push(entry.into()); + } + + /// Annotate `text` by appending ` [?Canonical]` after any word-token + /// that looks like a misspelling of a gazetteer entry. Non-matching + /// tokens and all non-word segments (whitespace, punctuation, digits) + /// pass through byte-identical. + pub fn annotate(&self, text: &str) -> String { + if self.is_empty() { + return text.to_string(); + } + let mut out = String::with_capacity(text.len()); + for seg in tokenize(text) { + match seg { + Segment::Word(w) => match self.find_candidate(w) { + Some(c) => { + out.push_str(w); + out.push_str(" [?"); + out.push_str(c); + out.push(']'); + } + None => out.push_str(w), + }, + Segment::Other(s) => out.push_str(s), + } + } + out + } + + fn find_candidate(&self, token: &str) -> Option<&str> { + if token.chars().count() < MIN_TOKEN_LEN { + return None; + } + let lower = token.to_lowercase(); + if self.exact.contains(lower.as_str()) { + return None; + } + let code = koelner::encode(token); + if code.is_empty() { + return None; + } + let candidates = self.by_phonetic.get(&code)?; + candidates + .iter() + .map(|c| (damerau_levenshtein(&lower, &c.to_lowercase()), c)) + .filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE) + .min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.as_ref().cmp(b.1.as_ref()))) + .map(|(_, c)| c.as_ref()) + } +} + +/// A slice of the input text — either a word or a non-word run. +enum Segment<'a> { + Word(&'a str), + Other(&'a str), +} + +/// Split `text` into alternating runs of alphabetic and non-alphabetic +/// chars. Round-trip: concatenating every segment yields the original +/// string byte-for-byte. +fn tokenize(text: &str) -> Vec> { + let mut out = Vec::new(); + let mut chars = text.char_indices().peekable(); + while let Some((start, c)) = chars.next() { + let is_alpha = c.is_alphabetic(); + let mut end = start + c.len_utf8(); + while let Some(&(next_start, next_c)) = chars.peek() { + if next_c.is_alphabetic() != is_alpha { + break; + } + end = next_start + next_c.len_utf8(); + chars.next(); + } + let slice = &text[start..end]; + out.push(if is_alpha { + Segment::Word(slice) + } else { + Segment::Other(slice) + }); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + /// Build a gazetteer from an in-memory list of canonical entries — + /// avoids filesystem I/O in the matcher tests. + fn from_entries(entries: &[&str]) -> Gazetteer { + let mut g = Gazetteer::empty(); + for e in entries { + g.insert(e); + } + g + } + + #[test] + fn empty_gazetteer_is_pass_through() { + let g = Gazetteer::empty(); + let input = "Patient nahm Womax gegen Übelkeit."; + assert_eq!(g.annotate(input), input); + } + + /// Core regression: the Womax/Vomex use case. A token that's a + /// phonetic neighbor at edit distance ≤ 2 must get a hint appended. + #[test] + fn annotates_typo_for_phonetic_neighbor() { + let g = from_entries(&["Vomex"]); + let out = g.annotate("Patient nahm Womax gegen Übelkeit."); + assert!( + out.contains("Womax [?Vomex]"), + "expected annotation, got: {out:?}" + ); + } + + #[test] + fn noop_on_exact_match() { + let g = from_entries(&["Vomex"]); + assert_eq!( + g.annotate("Patient nahm Vomex."), + "Patient nahm Vomex.", + ); + } + + /// Lowercase variant of an entry is still "exact" — we never nag + /// the LLM with a hint for a token that's only case-different. + #[test] + fn noop_on_case_only_difference() { + let g = from_entries(&["Vomex"]); + assert_eq!( + g.annotate("Patient nahm vomex."), + "Patient nahm vomex.", + ); + } + + #[test] + fn noop_on_unrelated_token() { + let g = from_entries(&["Vomex"]); + let input = "Patient hat Husten und Fieber."; + assert_eq!(g.annotate(input), input); + } + + /// Tokens shorter than MIN_TOKEN_LEN are never annotated, even if + /// the gazetteer contains a matching short entry. + #[test] + fn noop_on_short_token() { + let g = from_entries(&["ACE"]); + let input = "ACE und AGE und APE"; + assert_eq!(g.annotate(input), input); + } + + /// With two phonetically-colliding candidates, the one with smaller + /// Damerau-Levenshtein distance wins. Womax↔Wamax is 1, + /// Womax↔Vomex is 2 → Wamax is chosen. + #[test] + fn picks_closest_candidate_by_edit_distance() { + let g = from_entries(&["Vomex", "Wamax"]); + let out = g.annotate("Patient nahm Womax."); + assert!( + out.contains("Womax [?Wamax]"), + "expected Wamax as closer candidate, got: {out:?}" + ); + } + + #[test] + fn preserves_whitespace_and_punctuation() { + let g = from_entries(&["Zebra"]); + let input = "Line 1.\n\nLine 2: foo-bar; baz!"; + assert_eq!(g.annotate(input), input); + } + + #[tokio::test] + async fn load_dir_reads_all_txt_files() { + let dir = tempdir().unwrap(); + tokio::fs::write(dir.path().join("meds.txt"), "Vomex\nAspirin\n") + .await + .unwrap(); + tokio::fs::write(dir.path().join("anatomy.txt"), "Coracoideus\n") + .await + .unwrap(); + let g = Gazetteer::load_dir(dir.path()).await.unwrap(); + assert_eq!(g.len(), 3); + } + + #[tokio::test] + async fn load_dir_skips_blank_lines_and_comments() { + let dir = tempdir().unwrap(); + tokio::fs::write( + dir.path().join("v.txt"), + "# medications\nVomex\n\n \n# another\nAspirin\n", + ) + .await + .unwrap(); + let g = Gazetteer::load_dir(dir.path()).await.unwrap(); + assert_eq!(g.len(), 2); + } + + #[tokio::test] + async fn load_dir_ignores_non_txt_files() { + let dir = tempdir().unwrap(); + tokio::fs::write(dir.path().join("v.txt"), "Vomex\n") + .await + .unwrap(); + tokio::fs::write(dir.path().join("readme.md"), "Aspirin\n") + .await + .unwrap(); + let g = Gazetteer::load_dir(dir.path()).await.unwrap(); + assert_eq!(g.len(), 1); + } + + #[tokio::test] + async fn load_dir_missing_dir_is_err() { + let dir = tempdir().unwrap(); + let missing = dir.path().join("nonexistent"); + assert!(Gazetteer::load_dir(&missing).await.is_err()); + } +}