diff --git a/src/eval.rs b/src/eval.rs index 3039f5d..2c2730f 100644 --- a/src/eval.rs +++ b/src/eval.rs @@ -2,7 +2,7 @@ use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; /// Inject realistic transcription errors deterministically (seeded): -/// char drop, adjacent swap, umlaut flattening, digit transposition. +/// char drop, adjacent swap, umlaut→ASCII-digraph (ä→ae), digit transposition. pub fn inject_errors(text: &str, rate: f64, seed: u64) -> String { let mut rng = StdRng::seed_from_u64(seed); let mut chars: Vec = text.chars().collect(); @@ -13,11 +13,18 @@ pub fn inject_errors(text: &str, rate: f64, seed: u64) -> String { match rng.gen_range(0..4) { 0 => { chars.remove(i); continue; } // drop 1 => { if i + 1 < chars.len() { chars.swap(i, i + 1); } } // swap - 2 => { // flatten umlaut - chars[i] = match chars[i] { - 'ä' => 'a', 'ö' => 'o', 'ü' => 'u', 'ß' => 's', - other => other, + 2 => { // umlaut → ASCII digraph (ä→ae) + let digraph = match chars[i] { + 'ä' => Some(['a', 'e']), 'ö' => Some(['o', 'e']), + 'ü' => Some(['u', 'e']), 'ß' => Some(['s', 's']), + _ => None, }; + if let Some([a, b]) = digraph { + chars[i] = a; + chars.insert(i + 1, b); + i += 2; + continue; + } } _ => { // digit transpose if chars[i].is_ascii_digit() && i + 1 < chars.len() diff --git a/src/normalize.rs b/src/normalize.rs index ab13d8e..5f3024f 100644 --- a/src/normalize.rs +++ b/src/normalize.rs @@ -9,8 +9,28 @@ const ABBREV: &[(&str, &str)] = &[ ("li.", "links"), ]; +/// Fold German umlauts and eszett to their ASCII digraph equivalents so that +/// "Hyperlipidämie" and "Hyperlipidaemie" collapse to one canonical token. +/// Operates on already-lowercased text (so only the lowercase forms appear). +/// Applied to BOTH the indexed corpus text and the query (see lexical.rs), so +/// the lexical path stays symmetric under either spelling. NOT applied to the +/// embedding path — bge-m3 is multilingual and handles umlauts natively. +fn fold_umlauts(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + 'ä' => out.push_str("ae"), + 'ö' => out.push_str("oe"), + 'ü' => out.push_str("ue"), + 'ß' => out.push_str("ss"), + _ => out.push(c), + } + } + out +} + pub fn normalize(input: &str) -> String { - let lower = input.to_lowercase(); + let lower = fold_umlauts(&input.to_lowercase()); let mut out: Vec = Vec::new(); for tok in lower.split_whitespace() { let replaced = ABBREV.iter() diff --git a/tests/normalize_tests.rs b/tests/normalize_tests.rs index a4f658f..822ca0d 100644 --- a/tests/normalize_tests.rs +++ b/tests/normalize_tests.rs @@ -11,7 +11,29 @@ fn expands_common_medical_abbreviations() { assert_eq!(normalize("Z.n. OP"), "zustand nach operation"); } +// The system must match regardless of whether the dictation uses real +// umlauts (ä, ö, ü, ß) or their ASCII digraphs (ae, oe, ue, ss). Both the +// indexed corpus text and the query pass through normalize(), so folding +// both forms to one canonical (ASCII-digraph) representation makes the +// lexical path symmetric — "Hyperlipidämie" and "Hyperlipidaemie" match. #[test] -fn keeps_german_umlauts() { - assert_eq!(normalize("Ödem"), "ödem"); +fn folds_umlauts_and_eszett_to_ascii_digraphs() { + assert_eq!(normalize("Ödem"), "oedem"); + assert_eq!(normalize("Fußödem"), "fussoedem"); + assert_eq!(normalize("über"), "ueber"); +} + +#[test] +fn umlaut_and_digraph_forms_normalize_identically() { + assert_eq!(normalize("Hyperlipidämie"), normalize("Hyperlipidaemie")); + assert_eq!(normalize("Ödem"), normalize("Oedem")); + assert_eq!(normalize("Fußödem"), normalize("Fussoedem")); + assert_eq!(normalize("über"), normalize("ueber")); +} + +#[test] +fn folding_is_idempotent_on_already_ascii_text() { + // ASCII-only medical text must be unchanged by the fold. + assert_eq!(normalize("hyperlipidaemie"), "hyperlipidaemie"); + assert_eq!(normalize("Diabetes Mellitus"), "diabetes mellitus"); }