diff --git a/server/Cargo.lock b/server/Cargo.lock index 9e9c147..1d55659 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -345,6 +345,7 @@ dependencies = [ "rpassword", "serde", "serde_json", + "spellbook", "strsim", "tempfile", "time", @@ -1631,6 +1632,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spellbook" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28332bb44a9e7e7f1aa43ce4d97b6d1b27323a1fc160d4cfe0089245e1750845" +dependencies = [ + "foldhash", + "hashbrown 0.17.0", +] + [[package]] name = "spin" version = "0.9.8" diff --git a/server/Cargo.toml b/server/Cargo.toml index ebadddd..27aea49 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -27,6 +27,7 @@ toml_edit = "0.25.11" rpassword = "7.4.0" pulldown-cmark = { version = "0.13", default-features = false, features = ["html"] } strsim = "0.11" +spellbook = "0.4.0" [dev-dependencies] tower = { version = "0.5", features = ["util"] } diff --git a/server/src/config.rs b/server/src/config.rs index 0c09626..370a578 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -62,6 +62,12 @@ pub struct Config { /// medications). Loaded once at startup; a missing directory is a /// non-fatal WARN — analysis runs without proper-name correction. pub vocab_dir: PathBuf, + /// Path prefix (without extension) to a hunspell dictionary pair + /// `.aff` + `.dic`. Used by the gazetteer as a dict + /// veto against false-positive rewrites (e.g. `Kaktus` → `Lantus`). + /// Missing / unreadable files are a non-fatal WARN — the gazetteer + /// falls back to dict-less behaviour. + pub hunspell_dict_path: PathBuf, pub session_timeout_hours: u32, /// Whether to set the `Secure` flag on the session cookie. Default `true`. /// Set `COOKIE_SECURE=false` only for plain-HTTP local development — @@ -105,6 +111,10 @@ impl Config { } }, vocab_dir: PathBuf::from(optional_env("VOCAB_DIR", "./vocab")), + hunspell_dict_path: PathBuf::from(optional_env( + "HUNSPELL_DICT", + "/usr/share/hunspell/de_DE", + )), session_timeout_hours: optional_env_parsed("SESSION_TIMEOUT_HOURS", 8), cookie_secure: optional_env_parsed("COOKIE_SECURE", true), } @@ -158,6 +168,7 @@ impl Config { llm_timeout_seconds: 180, llm_system_prompt: crate::analyze::prompt::SYSTEM_PROMPT.to_string(), vocab_dir: PathBuf::new(), + hunspell_dict_path: PathBuf::new(), session_timeout_hours: 8, cookie_secure: false, } diff --git a/server/src/gazetteer/mod.rs b/server/src/gazetteer/mod.rs index 6ec2ea3..f0928b1 100644 --- a/server/src/gazetteer/mod.rs +++ b/server/src/gazetteer/mod.rs @@ -5,8 +5,8 @@ //! Invariant: every AI-produced text passes through `replace()` before //! being written to disk or forwarded downstream. Tokens within //! edit-distance 2 of a curated vocabulary entry are silently rewritten -//! to the canonical form. No hints, no annotations, no LLM veto — the -//! vocab is authoritative. +//! to the canonical form — *unless* the token itself is a valid word +//! of the target language (dict-veto), in which case it is preserved. //! //! Rationale: LLMs have strong training priors that override contextual //! hints — e.g. they anglicize `Amiodaron` to `Amiodarone` even when @@ -19,8 +19,16 @@ //! entries, filtered by distance ≤ MAX_EDIT_DISTANCE. At typical sizes //! (~200 entries × ~50 tokens), the per-pass cost stays well under a //! millisecond — negligible next to the AI call it follows. +//! +//! Dict-veto: if a token is within the edit-distance window of a +//! vocab entry but is itself a valid word of the language (checked +//! via a pluggable `DictChecker`, e.g. hunspell/spellbook), we keep +//! it. Prevents false positives like `Kaktus` → `Lantus` (DL=2). +//! Lookup is lazy: only called once we already have a DL-candidate, +//! so common words (99% of a transcript) never touch the dict. use std::collections::HashSet; +use std::fmt; use std::io; use std::path::Path; @@ -38,10 +46,19 @@ const MIN_TOKEN_LEN: usize = 5; /// short-circuit; distances > 2 are rejected as too speculative. const MAX_EDIT_DISTANCE: usize = 2; +/// Veto gate for `replace()`: if `check()` returns true for a token, +/// the token is considered a valid word of the language and is +/// preserved unchanged, even when a DL-close vocab entry exists. +/// Implementations must be `Send + Sync` so a `Gazetteer` can be +/// shared across async tasks via `Arc`. +pub trait DictChecker: Send + Sync { + fn check(&self, word: &str) -> bool; +} + /// Domain-specific vocabulary, populated once at server start from /// `*.txt` files under a vocab directory. Read-only thereafter; share /// via `Arc`. -#[derive(Debug, Default)] +#[derive(Default)] pub struct Gazetteer { /// Canonical entries in original casing — source of truth for both /// scanning and emission. @@ -49,6 +66,19 @@ pub struct Gazetteer { /// Lowercased canonicals; short-circuits replacement when the token /// already matches a known entry exactly. exact: HashSet>, + /// Optional language-word veto. When set, a DL-candidate token + /// that checks as a real word is left untouched. `None` = old + /// behaviour (replace every DL-candidate). + dict: Option>, +} + +impl fmt::Debug for Gazetteer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Gazetteer") + .field("entries", &self.entries.len()) + .field("has_dict", &self.dict.is_some()) + .finish() + } } impl Gazetteer { @@ -87,6 +117,16 @@ impl Gazetteer { self.entries.is_empty() } + /// Attach a language-word checker. The returned `Gazetteer` vetoes + /// replacements whose input token is a recognized word of the + /// language (e.g. `Kaktus` stays `Kaktus` even though it's DL=2 + /// from `Lantus`). Builder-style so `load_dir(...).await?.with_dict(d)` + /// is a single expression. + pub fn with_dict(mut self, dict: Box) -> Self { + self.dict = Some(dict); + self + } + fn insert(&mut self, entry: &str) { if entry.chars().count() < MIN_TOKEN_LEN { return; @@ -138,12 +178,79 @@ impl Gazetteer { return None; } - self.entries + let best = self + .entries .iter() .map(|c| (damerau_levenshtein(&lower, &c.to_lowercase()), c.as_ref())) .filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE) - .min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(b.1))) - .map(|(d, c)| (c, d)) + .min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(b.1)))?; + + // Dict-veto: only consult the checker *after* we already have + // a DL-candidate. Common words (99% of a transcript) never + // get here, so the dict cost is paid only on the ~0.1% of + // tokens that are actually at risk of mis-replacement. + // + // Case handling: German nouns are capitalized and hunspell + // rejects them when lowercased. We try the token as-is, then + // with a capitalized first char (in case Whisper lowercased + // a noun at sentence start). Either match is a veto. + if let Some(dict) = self.dict.as_ref() { + let first_upper = first_char_uppercased(token); + if dict.check(token) || dict.check(&first_upper) { + tracing::info!( + token = %token, + candidate = best.1, + distance = best.0, + "gazetteer blocked by dict veto", + ); + return None; + } + } + + Some((best.1, best.0)) + } +} + +/// Return `token` with its first character uppercased; rest untouched. +/// Cheap: allocates only when the first char actually changes case. +/// Example: `"kaktus"` → `"Kaktus"`, `"Kaktus"` → `"Kaktus"`. +fn first_char_uppercased(token: &str) -> String { + let mut chars = token.chars(); + match chars.next() { + Some(first) => { + let mut out: String = first.to_uppercase().collect(); + out.push_str(chars.as_str()); + out + } + None => String::new(), + } +} + +/// spellbook-backed `DictChecker`. Loads hunspell-format `.aff` + +/// `.dic` once at construction; applies affix expansion on every +/// `check()` call, so flected forms (`Kakteen`, `Kaktusse`) are +/// recognized even though only `Kaktus` lives in the `.dic`. +pub struct SpellbookDict { + inner: spellbook::Dictionary, +} + +impl SpellbookDict { + /// `stem` is the path prefix: e.g. `/usr/share/hunspell/de_DE`. + /// Both `.aff` and `.dic` must exist and be readable. + pub fn load(stem: &Path) -> io::Result { + let aff_path = stem.with_extension("aff"); + let dic_path = stem.with_extension("dic"); + let aff = std::fs::read_to_string(&aff_path)?; + let dic = std::fs::read_to_string(&dic_path)?; + let inner = spellbook::Dictionary::new(&aff, &dic) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?; + Ok(Self { inner }) + } +} + +impl DictChecker for SpellbookDict { + fn check(&self, word: &str) -> bool { + self.inner.check(word) } } @@ -192,6 +299,24 @@ mod tests { g } + /// In-memory `DictChecker` for unit tests. **Case-sensitive** on + /// purpose — mirrors how hunspell-de treats nouns (`Kaktus` in the + /// dict, `kaktus` rejected). This ensures our tests exercise the + /// same path the production code takes with real hunspell. + struct HashSetDict(HashSet); + + impl HashSetDict { + fn new(words: &[&str]) -> Self { + Self(words.iter().map(|w| w.to_string()).collect()) + } + } + + impl DictChecker for HashSetDict { + fn check(&self, word: &str) -> bool { + self.0.contains(word) + } + } + #[test] fn empty_gazetteer_is_pass_through() { let g = Gazetteer::empty(); @@ -337,4 +462,95 @@ mod tests { let missing = dir.path().join("nonexistent"); assert!(Gazetteer::load_dir(&missing).await.is_err()); } + + /// Core false-positive fix: `Kaktus` is an everyday German word + /// that sits at DL=2 from `Lantus`. Without the dict veto, the + /// naive DL rule would rewrite it. With the veto, it stays. + /// Mock dict stores `Kaktus` case-sensitive — so the original + /// token matches directly without any case-folding. + #[test] + fn dict_veto_blocks_alltagswort_rewrite() { + let g = from_entries(&["Lantus"]).with_dict(Box::new(HashSetDict::new(&["Kaktus"]))); + assert_eq!( + g.replace("Patient hat einen Kaktus."), + "Patient hat einen Kaktus." + ); + } + + /// Case-robustness regression: Whisper sometimes emits a noun + /// lowercase (sentence start missing, ASR artefacts). Even then, + /// our first-char-uppercase fallback hits the dict and the veto + /// holds. Mock contains the canonical `Kaktus`, input is the + /// lowercased variant — replacement must still be blocked. + #[test] + fn dict_veto_handles_lowercased_noun_via_first_char_upper() { + let g = from_entries(&["Lantus"]).with_dict(Box::new(HashSetDict::new(&["Kaktus"]))); + assert_eq!(g.replace("ein kaktus steht da."), "ein kaktus steht da."); + } + + /// Negative control: a token that is *not* a recognized word + /// still gets rewritten. `Zerebrum` is a typo of `Cerebrum` + /// (it's the German spelling people reach for but the canonical + /// vocab form uses `C`). Dict does not contain `zerebrum`. + #[test] + fn dict_veto_allows_nonword_rewrite() { + let g = from_entries(&["Cerebrum"]).with_dict(Box::new(HashSetDict::new(&["Kaktus"]))); + assert_eq!( + g.replace("Blutung im Zerebrum."), + "Blutung im Cerebrum." + ); + } + + /// The exact-match short-circuit runs *before* any dict lookup. + /// Even if the dict also contained `Lantus`, we'd never get to + /// the veto — but more importantly, an exact hit should just + /// pass through cheaply. + #[test] + fn dict_veto_preserves_exact_match_fast_path() { + let g = from_entries(&["Lantus"]).with_dict(Box::new(HashSetDict::new(&["Lantus"]))); + assert_eq!(g.replace("Patient nimmt Lantus."), "Patient nimmt Lantus."); + } + + /// Regression guard: without a dict attached, the gazetteer keeps + /// its historical behaviour — the dict field defaults to None. + #[test] + fn gazetteer_without_dict_behaves_as_before() { + let g = from_entries(&["Lantus"]); + assert_eq!( + g.replace("Patient hat einen Kaktus."), + "Patient hat einen Lantus." + ); + } + + /// Live check against the system-installed hunspell-de. + /// Marked `#[ignore]` because it depends on + /// `/usr/share/hunspell/de_DE.{aff,dic}` being present — the + /// regular test suite must pass on any build host. Run via + /// `cargo test -- --ignored` on a machine with `pacman -S + /// hunspell-de` (or the equivalent). + /// + /// Proves two things at once: + /// 1. spellbook can actually parse the German .aff/.dic pair + /// (the README warns about complex compound directives). + /// 2. Affix expansion is applied — the canonical plural + /// `Kakteen` of the base form `Kaktus` is recognized even + /// though `Kakteen` is not literally in the .dic. + #[test] + #[ignore] + fn spellbook_dict_recognizes_german_base_and_inflection() { + let dict = SpellbookDict::load(Path::new("/usr/share/hunspell/de_DE")) + .expect("hunspell-de installed"); + assert!(dict.check("Kaktus"), "base form should be in dict"); + assert!(dict.check("Patient"), "common word should be in dict"); + assert!(!dict.check("Lantus"), "medical brand should NOT be in dict"); + assert!(!dict.check("Zerebrum"), "misspelling should NOT be in dict"); + // The real payoff: canonical plural of Kaktus is Kakteen. + // If spellbook doesn't apply affix expansion for German, this + // will fail and we need to switch to a hunspell FFI crate. + assert!( + dict.check("Kakteen"), + "inflected form must be recognized via affix expansion" + ); + } + } diff --git a/server/src/main.rs b/server/src/main.rs index ced0aa8..20fca87 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -10,7 +10,7 @@ use tracing_subscriber::EnvFilter; use doctate_server::analyze; use doctate_server::config::Config; -use doctate_server::gazetteer::Gazetteer; +use doctate_server::gazetteer::{Gazetteer, SpellbookDict}; use doctate_server::transcribe; use doctate_server::AppState; @@ -69,7 +69,7 @@ async fn main() { // every AI output (Whisper, oneliner, analyze). Missing / unreadable // vocab directory is non-fatal — the pipeline runs without // normalization. - let vocab = Arc::new(match Gazetteer::load_dir(&config.vocab_dir).await { + let base_gazetteer = match Gazetteer::load_dir(&config.vocab_dir).await { Ok(g) => { info!( dir = %config.vocab_dir.display(), @@ -86,6 +86,28 @@ async fn main() { ); Gazetteer::empty() } + }; + + // Dict-veto: preserve tokens that are real German words, even if + // they sit at DL ≤ 2 from a vocab entry (e.g. `Kaktus` stays + // `Kaktus`, does not become `Lantus`). Missing / unreadable + // hunspell files are non-fatal — pipeline runs without the veto. + let vocab = Arc::new(match SpellbookDict::load(&config.hunspell_dict_path) { + Ok(dict) => { + info!( + stem = %config.hunspell_dict_path.display(), + "Hunspell dict loaded for gazetteer veto" + ); + base_gazetteer.with_dict(Box::new(dict)) + } + Err(e) => { + warn!( + stem = %config.hunspell_dict_path.display(), + error = %e, + "Hunspell dict unavailable; gazetteer running without dict veto" + ); + base_gazetteer + } }); // Transcription pipeline: channel + worker + recovery scan.