diff --git a/server/src/gazetteer/koelner.rs b/server/src/gazetteer/koelner.rs new file mode 100644 index 0000000..b259825 --- /dev/null +++ b/server/src/gazetteer/koelner.rs @@ -0,0 +1,238 @@ +//! Kölner Phonetik (Postel 1969) — deterministic sound-hash for German +//! words. Two inputs with the same code are phonetically similar. +//! +//! We use this to find gazetteer entries that *sound like* a +//! misspelled token in the transcript. Example: both `Vomex` (real +//! drug name) and `Womax` (Whisper hallucination) encode to `3648`, +//! so a gazetteer containing `Vomex` can surface it as a candidate +//! correction for `Womax`. +//! +//! Pure function, no I/O, no allocation beyond the returned `String`. + +/// Encode `s` as a Kölner-Phonetik code. Output is a sequence of ASCII +/// digits `0`–`8`. Non-alphabetic input chars are silently dropped; +/// case is ignored. Empty input returns an empty string. +/// +/// Rules (condensed from Postel 1969 / Wikipedia): +/// - Vowels `A E I J O U Y` → `0`; leading `0` is kept, internal / +/// trailing `0`s are removed in post-processing. +/// - `H` is not encoded at all (neither emitted nor collapsed). +/// - `B` → `1`; `P` → `1`, but `PH` → `3`. +/// - `D T` → `2`, but before `C S Z` → `8`. +/// - `F V W` → `3`. +/// - `G K Q` → `4`. +/// - `C` is context-dependent: +/// * initial + before `A H K L O Q R U X` → `4` +/// * initial otherwise → `8` +/// * after `S Z` → `8` +/// * before `A H K O Q U X` → `4` +/// * otherwise → `8` +/// - `X` → `48`, but after `C K Q` → `8` (those already encoded the 4). +/// - `L` → `5`; `M N` → `6`; `R` → `7`; `S Z` → `8`. +/// - Post: collapse consecutive equal digits, then drop every non-leading `0`. +pub fn encode(s: &str) -> String { + let prepared = prepare(s); + if prepared.is_empty() { + return String::new(); + } + let raw = raw_encode(&prepared); + postprocess(&raw) +} + +/// Expand umlauts + `ß`, uppercase everything else, drop non-A-Z chars. +/// Returns a `Vec` because subsequent passes need random access +/// and look-ahead, which is awkward on `&str` (UTF-8 byte indices ≠ +/// char positions). +fn prepare(s: &str) -> Vec { + let mut out: Vec = Vec::with_capacity(s.len()); + for c in s.chars() { + match c { + 'ä' | 'Ä' => out.push('A'), + 'ö' | 'Ö' => out.push('O'), + 'ü' | 'Ü' => out.push('U'), + 'ß' => { + out.push('S'); + out.push('S'); + } + _ => { + for u in c.to_uppercase() { + if u.is_ascii_uppercase() { + out.push(u); + } + } + } + } + } + out +} + +/// Per-character encoding with one-char look-ahead and look-behind. +/// Emits ASCII digit chars into the output buffer. `H` emits nothing; +/// `X` may emit two chars. +fn raw_encode(chars: &[char]) -> Vec { + let mut out: Vec = Vec::with_capacity(chars.len() + 2); + for (i, &c) in chars.iter().enumerate() { + let next = chars.get(i + 1).copied(); + let prev = if i > 0 { chars.get(i - 1).copied() } else { None }; + match c { + 'A' | 'E' | 'I' | 'J' | 'O' | 'U' | 'Y' => out.push('0'), + 'H' => {} // not encoded + 'B' => out.push('1'), + 'P' => out.push(if next == Some('H') { '3' } else { '1' }), + 'D' | 'T' => out.push( + if matches!(next, Some('C') | Some('S') | Some('Z')) { + '8' + } else { + '2' + }, + ), + 'F' | 'V' | 'W' => out.push('3'), + 'G' | 'K' | 'Q' => out.push('4'), + 'C' => { + let d = if i == 0 { + // Word-initial C. + if matches!( + next, + Some('A') | Some('H') | Some('K') | Some('L') + | Some('O') | Some('Q') | Some('R') + | Some('U') | Some('X') + ) { + '4' + } else { + '8' + } + } else if matches!(prev, Some('S') | Some('Z')) { + // Override: after S/Z always 8, regardless of next. + '8' + } else if matches!( + next, + Some('A') | Some('H') | Some('K') | Some('O') + | Some('Q') | Some('U') | Some('X') + ) { + '4' + } else { + '8' + }; + out.push(d); + } + 'X' => { + if matches!(prev, Some('C') | Some('K') | Some('Q')) { + // Previous char already emitted the '4'; only the '8' is new. + out.push('8'); + } else { + out.push('4'); + out.push('8'); + } + } + 'L' => out.push('5'), + 'M' | 'N' => out.push('6'), + 'R' => out.push('7'), + 'S' | 'Z' => out.push('8'), + _ => {} // unreachable after prepare(), but harmless. + } + } + out +} + +/// Collapse runs of identical digits into one, then drop every internal +/// or trailing `0`. A leading `0` is preserved because it carries +/// information — the word started with a vowel. +fn postprocess(raw: &[char]) -> String { + let mut dedup: Vec = Vec::with_capacity(raw.len()); + for &c in raw { + if dedup.last() != Some(&c) { + dedup.push(c); + } + } + let mut out = String::with_capacity(dedup.len()); + for (i, &c) in dedup.iter().enumerate() { + if c == '0' && i > 0 { + continue; + } + out.push(c); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn encodes_classic_german_surnames() { + // Reference values from the German Wikipedia article on Kölner Phonetik. + assert_eq!(encode("Müller"), "657"); + assert_eq!(encode("Meier"), "67"); + assert_eq!(encode("Meyer"), "67"); + assert_eq!(encode("Mayer"), "67"); + assert_eq!(encode("Schmidt"), "862"); + assert_eq!(encode("Schmitz"), "868"); + } + + #[test] + fn treats_umlaut_and_digraph_as_equivalent() { + // Classic test: "ü" and "ue" should normalise to the same code. + assert_eq!(encode("Müller"), encode("Mueller")); + } + + /// Regression guard for the ASR-correction use case. Womax (Whisper + /// hallucination) and Vomex (real drug name) must encode identically + /// — that is the mechanism that makes the gazetteer lookup fire. + #[test] + fn vomex_and_womax_collide() { + assert_eq!(encode("Vomex"), encode("Womax")); + assert_eq!(encode("Vomex"), "3648"); + } + + #[test] + fn empty_and_non_alphabetic_input_yields_empty_code() { + assert_eq!(encode(""), ""); + assert_eq!(encode("123 !?"), ""); + } + + #[test] + fn is_case_insensitive() { + assert_eq!(encode("MÜLLER"), encode("müller")); + assert_eq!(encode("MüLLeR"), encode("Müller")); + } + + #[test] + fn leading_vowel_keeps_leading_zero() { + // "Aber" → A=0, B=1, E=0, R=7 → "0107" → dedupe: "0107" → drop non-leading 0: "017". + assert_eq!(encode("Aber"), "017"); + } + + #[test] + fn c_before_h_at_word_start_codes_4() { + // "Christian": C(initial,next=H→4) H(skip) R=7 I=0 S=8 T(next=I,not CSZ→2) I=0 A=0 N=6 + // raw = 4,7,0,8,2,0,0,6 → dedupe = 4,7,0,8,2,0,6 → drop internal 0 = 4,7,8,2,6 + assert_eq!(encode("Christian"), "47826"); + } + + #[test] + fn x_after_c_is_single_digit() { + // "Axel": A=0, X (prev=A, emits 4+8), E=0, L=5 → 0,4,8,0,5 → dedupe same → "0485" + assert_eq!(encode("Axel"), "0485"); + // "Knixel": K=4, N=6, I=0, X (prev=I, emits 4+8), E=0, L=5 → 4,6,0,4,8,0,5 → "46485" + assert_eq!(encode("Knixel"), "46485"); + } + + #[test] + fn ignores_embedded_digits_and_punctuation() { + // "F1-Chip" → prepared: F,C,H,I,P. F=3, C(non-initial,prev=F,next=H→4), + // H(skip), I=0, P(no next=H→1). Raw: 3,4,0,1 → no dedupe → "341". + assert_eq!(encode("F1-Chip"), "341"); + } + + #[test] + fn ph_at_start_codes_3() { + // "Phil" → P(next=H→3), H(skip), I=0, L=5 → 3,0,5 → "35". + assert_eq!(encode("Phil"), "35"); + } + + #[test] + fn s_z_before_c_yields_8() { + // "Katze" → K=4, A=0, T(next=Z→8), Z=8, E=0 → 4,0,8,8,0 → dedupe 4,0,8,0 → "48". + assert_eq!(encode("Katze"), "48"); + } +} diff --git a/server/src/gazetteer/mod.rs b/server/src/gazetteer/mod.rs new file mode 100644 index 0000000..549634b --- /dev/null +++ b/server/src/gazetteer/mod.rs @@ -0,0 +1,9 @@ +//! Gazetteer: Pre-LLM correction layer for proper names and technical +//! vocabulary (medications, anatomy, substances). See +//! `~/.claude/plans/happy-mapping-snail.md` for the design document. +//! +//! This module is currently a stub — only the Kölner Phonetik encoder +//! exists. The `Gazetteer` struct, loader, and annotator land in later +//! steps of the implementation. + +pub mod koelner;