Add gazetteer configuration and dependency

This commit introduces the `VOCAB_DIR` configuration option, which
specifies the directory for gazetteer files. It also adds the `strsim`
crate, which is used for calculating string similarity and is essential
for the gazetteer's functionality.

The gazetteer module has been significantly expanded to include the
`Gazetteer` struct, its loading mechanism from `.txt` files, and an
annotation function. This functionality allows for pre-LLM correction of
proper names and technical vocabulary by identifying potential
misspellings based on phonetic similarity and edit distance.

server/.env.example

@@ -33,5 +33,10 @@ LLM_TIMEOUT_SECONDS=180
 # Single line — escape newlines with \n if your shell preserves them, or put the prompt on one logical line.
 LLM_SYSTEM_PROMPT="Du bist ein medizinischer Assistent. Fasse die folgenden diktierten Abschnitte zu einem zusammenhängenden Text zusammen. Bereinigen und strukturieren, nichts hinzufügen, nichts interpretieren, keine Diagnose, keine Arztbrief-Struktur. Entferne Redundanzen. Spätere Aufnahmen haben Vorrang — Korrekturen, Nachträge und Widersprüche zugunsten der chronologisch letzten Aussage auflösen. Antworte auf Deutsch. Nur der zusammengefasste Text, keine Einleitung, kein Abschlusssatz."
 
+# Gazetteer (pre-LLM proper-name correction)
+# Directory with *.txt files, one entry per line. Blank and #-comment lines
+# are skipped. Missing directory is non-fatal — analysis runs without hints.
+VOCAB_DIR=./vocab
+
 # Session
 SESSION_TIMEOUT_HOURS=8

server/Cargo.lock

@@ -345,6 +345,7 @@ dependencies = [
  "rpassword",
  "serde",
  "serde_json",
+ "strsim",
  "tempfile",
  "time",
  "tokio",
@@ -1642,6 +1643,12 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
 
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
 [[package]]
 name = "subtle"
 version = "2.6.1"

server/Cargo.toml

@@ -26,6 +26,7 @@ time = { version = "0.3.47", features = ["local-offset", "parsing", "formatting"
 toml_edit = "0.25.11"
 rpassword = "7.4.0"
 pulldown-cmark = { version = "0.13", default-features = false, features = ["html"] }
+strsim = "0.11"
 
 [dev-dependencies]
 tower = { version = "0.5", features = ["util"] }

server/src/config.rs

@@ -58,6 +58,10 @@ pub struct Config {
     /// or empty. Runtime-configurable so admins can iterate and use the
     /// "Neu analysieren" button to re-run cases.
     pub llm_system_prompt: String,
+    /// Directory containing gazetteer `*.txt` files (anatomy, substances,
+    /// medications). Loaded once at startup; a missing directory is a
+    /// non-fatal WARN — analysis runs without proper-name correction.
+    pub vocab_dir: PathBuf,
     pub session_timeout_hours: u32,
     /// Whether to set the `Secure` flag on the session cookie. Default `true`.
     /// Set `COOKIE_SECURE=false` only for plain-HTTP local development —
@@ -100,6 +104,7 @@ impl Config {
                     v
                 }
             },
+            vocab_dir: PathBuf::from(optional_env("VOCAB_DIR", "./vocab")),
             session_timeout_hours: optional_env_parsed("SESSION_TIMEOUT_HOURS", 8),
             cookie_secure: optional_env_parsed("COOKIE_SECURE", true),
         }
@@ -152,6 +157,7 @@ impl Config {
             llm_temperature: 0.0,
             llm_timeout_seconds: 180,
             llm_system_prompt: crate::analyze::prompt::SYSTEM_PROMPT.to_string(),
+            vocab_dir: PathBuf::new(),
             session_timeout_hours: 8,
             cookie_secure: false,
         }

server/src/gazetteer/mod.rs

@@ -1,9 +1,311 @@
-//! Gazetteer: Pre-LLM correction layer for proper names and technical
-//! vocabulary (medications, anatomy, substances). See
+//! Gazetteer: domain-vocabulary lookup that injects correction hints
+//! before the analysis LLM runs. See
 //! `~/.claude/plans/happy-mapping-snail.md` for the design document.
 //!
-//! This module is currently a stub — only the Kölner Phonetik encoder
-//! exists. The `Gazetteer` struct, loader, and annotator land in later
-//! steps of the implementation.
+//! The LLM cannot know invented proper names (drug brands, anatomical
+//! Latin/Greek terms). Whisper misrenders them phonetically
+//! (`Vomex` → `Womax`). This module exposes a static lookup that, given
+//! a transcript, finds tokens that are likely typos of a known entry
+//! and rewrites them as `token [?Canonical]`. The LLM decides in context
+//! whether to adopt the hint.
 
 pub mod koelner;
+
+use std::collections::{HashMap, HashSet};
+use std::io;
+use std::path::Path;
+
+use strsim::damerau_levenshtein;
+
+/// Tokens below this char count are never annotated. Short tokens produce
+/// too many false positives (ACE, EKG, TAV).
+const MIN_TOKEN_LEN: usize = 4;
+
+/// Maximum Damerau-Levenshtein distance for a candidate to qualify as
+/// a typo of a gazetteer entry. Distance 0 is excluded — the `exact`
+/// set short-circuits that case.
+const MAX_EDIT_DISTANCE: usize = 2;
+
+/// Domain-specific vocabulary, populated once at server start from
+/// `*.txt` files under a vocab directory. Read-only thereafter; share
+/// via `Arc<Gazetteer>`.
+#[derive(Debug, Default)]
+pub struct Gazetteer {
+    /// Kölner-Phonetik code → canonical entries sharing that code. The
+    /// canonicals keep their original casing for display.
+    by_phonetic: HashMap<String, Vec<Box<str>>>,
+    /// Lowercased canonicals; short-circuits annotation when the token
+    /// is already spelled correctly.
+    exact: HashSet<Box<str>>,
+}
+
+impl Gazetteer {
+    /// Empty gazetteer. `annotate` is a pure pass-through on this instance.
+    pub fn empty() -> Self {
+        Self::default()
+    }
+
+    /// Load every `*.txt` file (non-recursive) under `dir`. Blank lines
+    /// and lines beginning with `#` are skipped. Case-insensitive
+    /// duplicates are silently collapsed.
+    pub async fn load_dir(dir: &Path) -> io::Result<Self> {
+        let mut g = Self::default();
+        let mut entries = tokio::fs::read_dir(dir).await?;
+        while let Some(entry) = entries.next_entry().await? {
+            let path = entry.path();
+            if path.extension().and_then(|e| e.to_str()) != Some("txt") {
+                continue;
+            }
+            let content = tokio::fs::read_to_string(&path).await?;
+            for line in content.lines() {
+                let trimmed = line.trim();
+                if trimmed.is_empty() || trimmed.starts_with('#') {
+                    continue;
+                }
+                g.insert(trimmed);
+            }
+        }
+        Ok(g)
+    }
+
+    /// Number of unique entries (case-insensitive).
+    pub fn len(&self) -> usize {
+        self.exact.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.exact.is_empty()
+    }
+
+    fn insert(&mut self, entry: &str) {
+        let lower = entry.to_lowercase();
+        if !self.exact.insert(lower.into_boxed_str()) {
+            // Duplicate (case-insensitive) — don't add to phonetic index either.
+            return;
+        }
+        let code = koelner::encode(entry);
+        if code.is_empty() {
+            return;
+        }
+        self.by_phonetic
+            .entry(code)
+            .or_default()
+            .push(entry.into());
+    }
+
+    /// Annotate `text` by appending ` [?Canonical]` after any word-token
+    /// that looks like a misspelling of a gazetteer entry. Non-matching
+    /// tokens and all non-word segments (whitespace, punctuation, digits)
+    /// pass through byte-identical.
+    pub fn annotate(&self, text: &str) -> String {
+        if self.is_empty() {
+            return text.to_string();
+        }
+        let mut out = String::with_capacity(text.len());
+        for seg in tokenize(text) {
+            match seg {
+                Segment::Word(w) => match self.find_candidate(w) {
+                    Some(c) => {
+                        out.push_str(w);
+                        out.push_str(" [?");
+                        out.push_str(c);
+                        out.push(']');
+                    }
+                    None => out.push_str(w),
+                },
+                Segment::Other(s) => out.push_str(s),
+            }
+        }
+        out
+    }
+
+    fn find_candidate(&self, token: &str) -> Option<&str> {
+        if token.chars().count() < MIN_TOKEN_LEN {
+            return None;
+        }
+        let lower = token.to_lowercase();
+        if self.exact.contains(lower.as_str()) {
+            return None;
+        }
+        let code = koelner::encode(token);
+        if code.is_empty() {
+            return None;
+        }
+        let candidates = self.by_phonetic.get(&code)?;
+        candidates
+            .iter()
+            .map(|c| (damerau_levenshtein(&lower, &c.to_lowercase()), c))
+            .filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE)
+            .min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.as_ref().cmp(b.1.as_ref())))
+            .map(|(_, c)| c.as_ref())
+    }
+}
+
+/// A slice of the input text — either a word or a non-word run.
+enum Segment<'a> {
+    Word(&'a str),
+    Other(&'a str),
+}
+
+/// Split `text` into alternating runs of alphabetic and non-alphabetic
+/// chars. Round-trip: concatenating every segment yields the original
+/// string byte-for-byte.
+fn tokenize(text: &str) -> Vec<Segment<'_>> {
+    let mut out = Vec::new();
+    let mut chars = text.char_indices().peekable();
+    while let Some((start, c)) = chars.next() {
+        let is_alpha = c.is_alphabetic();
+        let mut end = start + c.len_utf8();
+        while let Some(&(next_start, next_c)) = chars.peek() {
+            if next_c.is_alphabetic() != is_alpha {
+                break;
+            }
+            end = next_start + next_c.len_utf8();
+            chars.next();
+        }
+        let slice = &text[start..end];
+        out.push(if is_alpha {
+            Segment::Word(slice)
+        } else {
+            Segment::Other(slice)
+        });
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::tempdir;
+
+    /// Build a gazetteer from an in-memory list of canonical entries —
+    /// avoids filesystem I/O in the matcher tests.
+    fn from_entries(entries: &[&str]) -> Gazetteer {
+        let mut g = Gazetteer::empty();
+        for e in entries {
+            g.insert(e);
+        }
+        g
+    }
+
+    #[test]
+    fn empty_gazetteer_is_pass_through() {
+        let g = Gazetteer::empty();
+        let input = "Patient nahm Womax gegen Übelkeit.";
+        assert_eq!(g.annotate(input), input);
+    }
+
+    /// Core regression: the Womax/Vomex use case. A token that's a
+    /// phonetic neighbor at edit distance ≤ 2 must get a hint appended.
+    #[test]
+    fn annotates_typo_for_phonetic_neighbor() {
+        let g = from_entries(&["Vomex"]);
+        let out = g.annotate("Patient nahm Womax gegen Übelkeit.");
+        assert!(
+            out.contains("Womax [?Vomex]"),
+            "expected annotation, got: {out:?}"
+        );
+    }
+
+    #[test]
+    fn noop_on_exact_match() {
+        let g = from_entries(&["Vomex"]);
+        assert_eq!(
+            g.annotate("Patient nahm Vomex."),
+            "Patient nahm Vomex.",
+        );
+    }
+
+    /// Lowercase variant of an entry is still "exact" — we never nag
+    /// the LLM with a hint for a token that's only case-different.
+    #[test]
+    fn noop_on_case_only_difference() {
+        let g = from_entries(&["Vomex"]);
+        assert_eq!(
+            g.annotate("Patient nahm vomex."),
+            "Patient nahm vomex.",
+        );
+    }
+
+    #[test]
+    fn noop_on_unrelated_token() {
+        let g = from_entries(&["Vomex"]);
+        let input = "Patient hat Husten und Fieber.";
+        assert_eq!(g.annotate(input), input);
+    }
+
+    /// Tokens shorter than MIN_TOKEN_LEN are never annotated, even if
+    /// the gazetteer contains a matching short entry.
+    #[test]
+    fn noop_on_short_token() {
+        let g = from_entries(&["ACE"]);
+        let input = "ACE und AGE und APE";
+        assert_eq!(g.annotate(input), input);
+    }
+
+    /// With two phonetically-colliding candidates, the one with smaller
+    /// Damerau-Levenshtein distance wins. Womax↔Wamax is 1,
+    /// Womax↔Vomex is 2 → Wamax is chosen.
+    #[test]
+    fn picks_closest_candidate_by_edit_distance() {
+        let g = from_entries(&["Vomex", "Wamax"]);
+        let out = g.annotate("Patient nahm Womax.");
+        assert!(
+            out.contains("Womax [?Wamax]"),
+            "expected Wamax as closer candidate, got: {out:?}"
+        );
+    }
+
+    #[test]
+    fn preserves_whitespace_and_punctuation() {
+        let g = from_entries(&["Zebra"]);
+        let input = "Line 1.\n\nLine 2: foo-bar; baz!";
+        assert_eq!(g.annotate(input), input);
+    }
+
+    #[tokio::test]
+    async fn load_dir_reads_all_txt_files() {
+        let dir = tempdir().unwrap();
+        tokio::fs::write(dir.path().join("meds.txt"), "Vomex\nAspirin\n")
+            .await
+            .unwrap();
+        tokio::fs::write(dir.path().join("anatomy.txt"), "Coracoideus\n")
+            .await
+            .unwrap();
+        let g = Gazetteer::load_dir(dir.path()).await.unwrap();
+        assert_eq!(g.len(), 3);
+    }
+
+    #[tokio::test]
+    async fn load_dir_skips_blank_lines_and_comments() {
+        let dir = tempdir().unwrap();
+        tokio::fs::write(
+            dir.path().join("v.txt"),
+            "# medications\nVomex\n\n   \n# another\nAspirin\n",
+        )
+        .await
+        .unwrap();
+        let g = Gazetteer::load_dir(dir.path()).await.unwrap();
+        assert_eq!(g.len(), 2);
+    }
+
+    #[tokio::test]
+    async fn load_dir_ignores_non_txt_files() {
+        let dir = tempdir().unwrap();
+        tokio::fs::write(dir.path().join("v.txt"), "Vomex\n")
+            .await
+            .unwrap();
+        tokio::fs::write(dir.path().join("readme.md"), "Aspirin\n")
+            .await
+            .unwrap();
+        let g = Gazetteer::load_dir(dir.path()).await.unwrap();
+        assert_eq!(g.len(), 1);
+    }
+
+    #[tokio::test]
+    async fn load_dir_missing_dir_is_err() {
+        let dir = tempdir().unwrap();
+        let missing = dir.path().join("nonexistent");
+        assert!(Gazetteer::load_dir(&missing).await.is_err());
+    }
+}