fix: symmetric umlaut/ß folding in lexical path + eval digraph case

Smoke-testing invented German dictations exposed that `ae`/`oe`/`ue`
spellings collapsed lexical recall (e.g. "Hyperlipidaemie" → garbage,
"Hyperlipidämie" → correct). normalize() now folds ä→ae, ö→oe, ü→ue,
ß→ss; it is applied only in lexical.rs (both corpus index and query),
so the path is symmetric under either spelling. The embedding path
(raw text for bge-m3) is intentionally untouched.

eval.rs inject_errors case 2 changed from umlaut-deletion (ä→a) to the
realistic ASCII-digraph variant (ä→ae) so the eval harness exercises —
and guards against regression of — exactly this failure class.

TDD: failing tests first; normalize_tests 5/5, full suite 44/0.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

src/eval.rs

@@ -2,7 +2,7 @@ use rand::rngs::StdRng;
 use rand::{Rng, SeedableRng};
 
 /// Inject realistic transcription errors deterministically (seeded):
-/// char drop, adjacent swap, umlaut flattening, digit transposition.
+/// char drop, adjacent swap, umlaut→ASCII-digraph (ä→ae), digit transposition.
 pub fn inject_errors(text: &str, rate: f64, seed: u64) -> String {
     let mut rng = StdRng::seed_from_u64(seed);
     let mut chars: Vec<char> = text.chars().collect();
@@ -13,11 +13,18 @@ pub fn inject_errors(text: &str, rate: f64, seed: u64) -> String {
             match rng.gen_range(0..4) {
                 0 => { chars.remove(i); continue; }                 // drop
                 1 => { if i + 1 < chars.len() { chars.swap(i, i + 1); } } // swap
-                2 => {                                               // flatten umlaut
+                2 => {                                               // umlaut → ASCII digraph (ä→ae)
-                    chars[i] = match chars[i] {
+                    let digraph = match chars[i] {
-                        'ä' => 'a', 'ö' => 'o', 'ü' => 'u', 'ß' => 's',
+                        'ä' => Some(['a', 'e']), 'ö' => Some(['o', 'e']),
-                        other => other,
+                        'ü' => Some(['u', 'e']), 'ß' => Some(['s', 's']),
+                        _ => None,
                     };
+                    if let Some([a, b]) = digraph {
+                        chars[i] = a;
+                        chars.insert(i + 1, b);
+                        i += 2;
+                        continue;
+                    }
                 }
                 _ => {                                               // digit transpose
                     if chars[i].is_ascii_digit() && i + 1 < chars.len()

src/normalize.rs

@@ -9,8 +9,28 @@ const ABBREV: &[(&str, &str)] = &[
     ("li.", "links"),
 ];
 
+/// Fold German umlauts and eszett to their ASCII digraph equivalents so that
+/// "Hyperlipidämie" and "Hyperlipidaemie" collapse to one canonical token.
+/// Operates on already-lowercased text (so only the lowercase forms appear).
+/// Applied to BOTH the indexed corpus text and the query (see lexical.rs), so
+/// the lexical path stays symmetric under either spelling. NOT applied to the
+/// embedding path — bge-m3 is multilingual and handles umlauts natively.
+fn fold_umlauts(s: &str) -> String {
+    let mut out = String::with_capacity(s.len());
+    for c in s.chars() {
+        match c {
+            'ä' => out.push_str("ae"),
+            'ö' => out.push_str("oe"),
+            'ü' => out.push_str("ue"),
+            'ß' => out.push_str("ss"),
+            _ => out.push(c),
+        }
+    }
+    out
+}
+
 pub fn normalize(input: &str) -> String {
-    let lower = input.to_lowercase();
+    let lower = fold_umlauts(&input.to_lowercase());
     let mut out: Vec<String> = Vec::new();
     for tok in lower.split_whitespace() {
         let replaced = ABBREV.iter()

tests/normalize_tests.rs

@@ -11,7 +11,29 @@ fn expands_common_medical_abbreviations() {
     assert_eq!(normalize("Z.n. OP"), "zustand nach operation");
 }
 
+// The system must match regardless of whether the dictation uses real
+// umlauts (ä, ö, ü, ß) or their ASCII digraphs (ae, oe, ue, ss). Both the
+// indexed corpus text and the query pass through normalize(), so folding
+// both forms to one canonical (ASCII-digraph) representation makes the
+// lexical path symmetric — "Hyperlipidämie" and "Hyperlipidaemie" match.
 #[test]
-fn keeps_german_umlauts() {
+fn folds_umlauts_and_eszett_to_ascii_digraphs() {
-    assert_eq!(normalize("Ödem"), "ödem");
+    assert_eq!(normalize("Ödem"), "oedem");
+    assert_eq!(normalize("Fußödem"), "fussoedem");
+    assert_eq!(normalize("über"), "ueber");
+}
+
+#[test]
+fn umlaut_and_digraph_forms_normalize_identically() {
+    assert_eq!(normalize("Hyperlipidämie"), normalize("Hyperlipidaemie"));
+    assert_eq!(normalize("Ödem"), normalize("Oedem"));
+    assert_eq!(normalize("Fußödem"), normalize("Fussoedem"));
+    assert_eq!(normalize("über"), normalize("ueber"));
+}
+
+#[test]
+fn folding_is_idempotent_on_already_ascii_text() {
+    // ASCII-only medical text must be unchanged by the fold.
+    assert_eq!(normalize("hyperlipidaemie"), "hyperlipidaemie");
+    assert_eq!(normalize("Diabetes Mellitus"), "diabetes mellitus");
 }