Add Kölner Phonetik encoder
Implements the Kölner Phonetik algorithm for German word sound-encoding. This phonetic algorithm is used to find gazetteer entries that sound similar to a potentially misspelled token, enabling correction suggestions. The implementation includes: - `encode`: The main function that orchestrates the encoding process. - `prepare`: Prepares the input string by expanding umlauts, uppercasing, and filtering non-alphabetic characters. - `raw_encode`: Performs the character-by-character phonetic encoding with context-aware rules. - `postprocess`: Cleans up the raw output by collapsing duplicate digits and removing leading zeros. Includes unit tests to verify the correctness of the encoding for various German words and edge cases, ensuring phonetic similarity and handling of special character combinations.
This commit is contained in:
@@ -0,0 +1,238 @@
|
||||
//! Kölner Phonetik (Postel 1969) — deterministic sound-hash for German
|
||||
//! words. Two inputs with the same code are phonetically similar.
|
||||
//!
|
||||
//! We use this to find gazetteer entries that *sound like* a
|
||||
//! misspelled token in the transcript. Example: both `Vomex` (real
|
||||
//! drug name) and `Womax` (Whisper hallucination) encode to `3648`,
|
||||
//! so a gazetteer containing `Vomex` can surface it as a candidate
|
||||
//! correction for `Womax`.
|
||||
//!
|
||||
//! Pure function, no I/O, no allocation beyond the returned `String`.
|
||||
|
||||
/// Encode `s` as a Kölner-Phonetik code. Output is a sequence of ASCII
|
||||
/// digits `0`–`8`. Non-alphabetic input chars are silently dropped;
|
||||
/// case is ignored. Empty input returns an empty string.
|
||||
///
|
||||
/// Rules (condensed from Postel 1969 / Wikipedia):
|
||||
/// - Vowels `A E I J O U Y` → `0`; leading `0` is kept, internal /
|
||||
/// trailing `0`s are removed in post-processing.
|
||||
/// - `H` is not encoded at all (neither emitted nor collapsed).
|
||||
/// - `B` → `1`; `P` → `1`, but `PH` → `3`.
|
||||
/// - `D T` → `2`, but before `C S Z` → `8`.
|
||||
/// - `F V W` → `3`.
|
||||
/// - `G K Q` → `4`.
|
||||
/// - `C` is context-dependent:
|
||||
/// * initial + before `A H K L O Q R U X` → `4`
|
||||
/// * initial otherwise → `8`
|
||||
/// * after `S Z` → `8`
|
||||
/// * before `A H K O Q U X` → `4`
|
||||
/// * otherwise → `8`
|
||||
/// - `X` → `48`, but after `C K Q` → `8` (those already encoded the 4).
|
||||
/// - `L` → `5`; `M N` → `6`; `R` → `7`; `S Z` → `8`.
|
||||
/// - Post: collapse consecutive equal digits, then drop every non-leading `0`.
|
||||
pub fn encode(s: &str) -> String {
|
||||
let prepared = prepare(s);
|
||||
if prepared.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
let raw = raw_encode(&prepared);
|
||||
postprocess(&raw)
|
||||
}
|
||||
|
||||
/// Expand umlauts + `ß`, uppercase everything else, drop non-A-Z chars.
|
||||
/// Returns a `Vec<char>` because subsequent passes need random access
|
||||
/// and look-ahead, which is awkward on `&str` (UTF-8 byte indices ≠
|
||||
/// char positions).
|
||||
fn prepare(s: &str) -> Vec<char> {
|
||||
let mut out: Vec<char> = Vec::with_capacity(s.len());
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'ä' | 'Ä' => out.push('A'),
|
||||
'ö' | 'Ö' => out.push('O'),
|
||||
'ü' | 'Ü' => out.push('U'),
|
||||
'ß' => {
|
||||
out.push('S');
|
||||
out.push('S');
|
||||
}
|
||||
_ => {
|
||||
for u in c.to_uppercase() {
|
||||
if u.is_ascii_uppercase() {
|
||||
out.push(u);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Per-character encoding with one-char look-ahead and look-behind.
|
||||
/// Emits ASCII digit chars into the output buffer. `H` emits nothing;
|
||||
/// `X` may emit two chars.
|
||||
fn raw_encode(chars: &[char]) -> Vec<char> {
|
||||
let mut out: Vec<char> = Vec::with_capacity(chars.len() + 2);
|
||||
for (i, &c) in chars.iter().enumerate() {
|
||||
let next = chars.get(i + 1).copied();
|
||||
let prev = if i > 0 { chars.get(i - 1).copied() } else { None };
|
||||
match c {
|
||||
'A' | 'E' | 'I' | 'J' | 'O' | 'U' | 'Y' => out.push('0'),
|
||||
'H' => {} // not encoded
|
||||
'B' => out.push('1'),
|
||||
'P' => out.push(if next == Some('H') { '3' } else { '1' }),
|
||||
'D' | 'T' => out.push(
|
||||
if matches!(next, Some('C') | Some('S') | Some('Z')) {
|
||||
'8'
|
||||
} else {
|
||||
'2'
|
||||
},
|
||||
),
|
||||
'F' | 'V' | 'W' => out.push('3'),
|
||||
'G' | 'K' | 'Q' => out.push('4'),
|
||||
'C' => {
|
||||
let d = if i == 0 {
|
||||
// Word-initial C.
|
||||
if matches!(
|
||||
next,
|
||||
Some('A') | Some('H') | Some('K') | Some('L')
|
||||
| Some('O') | Some('Q') | Some('R')
|
||||
| Some('U') | Some('X')
|
||||
) {
|
||||
'4'
|
||||
} else {
|
||||
'8'
|
||||
}
|
||||
} else if matches!(prev, Some('S') | Some('Z')) {
|
||||
// Override: after S/Z always 8, regardless of next.
|
||||
'8'
|
||||
} else if matches!(
|
||||
next,
|
||||
Some('A') | Some('H') | Some('K') | Some('O')
|
||||
| Some('Q') | Some('U') | Some('X')
|
||||
) {
|
||||
'4'
|
||||
} else {
|
||||
'8'
|
||||
};
|
||||
out.push(d);
|
||||
}
|
||||
'X' => {
|
||||
if matches!(prev, Some('C') | Some('K') | Some('Q')) {
|
||||
// Previous char already emitted the '4'; only the '8' is new.
|
||||
out.push('8');
|
||||
} else {
|
||||
out.push('4');
|
||||
out.push('8');
|
||||
}
|
||||
}
|
||||
'L' => out.push('5'),
|
||||
'M' | 'N' => out.push('6'),
|
||||
'R' => out.push('7'),
|
||||
'S' | 'Z' => out.push('8'),
|
||||
_ => {} // unreachable after prepare(), but harmless.
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Collapse runs of identical digits into one, then drop every internal
|
||||
/// or trailing `0`. A leading `0` is preserved because it carries
|
||||
/// information — the word started with a vowel.
|
||||
fn postprocess(raw: &[char]) -> String {
|
||||
let mut dedup: Vec<char> = Vec::with_capacity(raw.len());
|
||||
for &c in raw {
|
||||
if dedup.last() != Some(&c) {
|
||||
dedup.push(c);
|
||||
}
|
||||
}
|
||||
let mut out = String::with_capacity(dedup.len());
|
||||
for (i, &c) in dedup.iter().enumerate() {
|
||||
if c == '0' && i > 0 {
|
||||
continue;
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn encodes_classic_german_surnames() {
|
||||
// Reference values from the German Wikipedia article on Kölner Phonetik.
|
||||
assert_eq!(encode("Müller"), "657");
|
||||
assert_eq!(encode("Meier"), "67");
|
||||
assert_eq!(encode("Meyer"), "67");
|
||||
assert_eq!(encode("Mayer"), "67");
|
||||
assert_eq!(encode("Schmidt"), "862");
|
||||
assert_eq!(encode("Schmitz"), "868");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn treats_umlaut_and_digraph_as_equivalent() {
|
||||
// Classic test: "ü" and "ue" should normalise to the same code.
|
||||
assert_eq!(encode("Müller"), encode("Mueller"));
|
||||
}
|
||||
|
||||
/// Regression guard for the ASR-correction use case. Womax (Whisper
|
||||
/// hallucination) and Vomex (real drug name) must encode identically
|
||||
/// — that is the mechanism that makes the gazetteer lookup fire.
|
||||
#[test]
|
||||
fn vomex_and_womax_collide() {
|
||||
assert_eq!(encode("Vomex"), encode("Womax"));
|
||||
assert_eq!(encode("Vomex"), "3648");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_and_non_alphabetic_input_yields_empty_code() {
|
||||
assert_eq!(encode(""), "");
|
||||
assert_eq!(encode("123 !?"), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_case_insensitive() {
|
||||
assert_eq!(encode("MÜLLER"), encode("müller"));
|
||||
assert_eq!(encode("MüLLeR"), encode("Müller"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leading_vowel_keeps_leading_zero() {
|
||||
// "Aber" → A=0, B=1, E=0, R=7 → "0107" → dedupe: "0107" → drop non-leading 0: "017".
|
||||
assert_eq!(encode("Aber"), "017");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn c_before_h_at_word_start_codes_4() {
|
||||
// "Christian": C(initial,next=H→4) H(skip) R=7 I=0 S=8 T(next=I,not CSZ→2) I=0 A=0 N=6
|
||||
// raw = 4,7,0,8,2,0,0,6 → dedupe = 4,7,0,8,2,0,6 → drop internal 0 = 4,7,8,2,6
|
||||
assert_eq!(encode("Christian"), "47826");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x_after_c_is_single_digit() {
|
||||
// "Axel": A=0, X (prev=A, emits 4+8), E=0, L=5 → 0,4,8,0,5 → dedupe same → "0485"
|
||||
assert_eq!(encode("Axel"), "0485");
|
||||
// "Knixel": K=4, N=6, I=0, X (prev=I, emits 4+8), E=0, L=5 → 4,6,0,4,8,0,5 → "46485"
|
||||
assert_eq!(encode("Knixel"), "46485");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_embedded_digits_and_punctuation() {
|
||||
// "F1-Chip" → prepared: F,C,H,I,P. F=3, C(non-initial,prev=F,next=H→4),
|
||||
// H(skip), I=0, P(no next=H→1). Raw: 3,4,0,1 → no dedupe → "341".
|
||||
assert_eq!(encode("F1-Chip"), "341");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ph_at_start_codes_3() {
|
||||
// "Phil" → P(next=H→3), H(skip), I=0, L=5 → 3,0,5 → "35".
|
||||
assert_eq!(encode("Phil"), "35");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s_z_before_c_yields_8() {
|
||||
// "Katze" → K=4, A=0, T(next=Z→8), Z=8, E=0 → 4,0,8,8,0 → dedupe 4,0,8,0 → "48".
|
||||
assert_eq!(encode("Katze"), "48");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
//! Gazetteer: Pre-LLM correction layer for proper names and technical
|
||||
//! vocabulary (medications, anatomy, substances). See
|
||||
//! `~/.claude/plans/happy-mapping-snail.md` for the design document.
|
||||
//!
|
||||
//! This module is currently a stub — only the Kölner Phonetik encoder
|
||||
//! exists. The `Gazetteer` struct, loader, and annotator land in later
|
||||
//! steps of the implementation.
|
||||
|
||||
pub mod koelner;
|
||||
Reference in New Issue
Block a user