diff --git a/server/src/gazetteer/mod.rs b/server/src/gazetteer/mod.rs index 7257b26..fa86db1 100644 --- a/server/src/gazetteer/mod.rs +++ b/server/src/gazetteer/mod.rs @@ -55,14 +55,33 @@ pub trait DictChecker: Send + Sync { fn check(&self, word: &str) -> bool; } +/// A single vocabulary entry: the canonical surface form plus an optional +/// set of explicit aliases that bypass the dict-veto on exact match. +/// +/// Bypass-aliases exist to override Hunspell's overly generous German +/// compound recognition. Example: `Sellink` is the canonical, but +/// Hunspell-de accepts `Seelink` as a valid `See`+`link` compound and +/// would veto the DL=1 replacement. Listing `seelink` in `bypass_aliases` +/// tells the gazetteer "trust me, this specific spelling is a known +/// mishearing of the canonical, skip the dict check". +/// +/// The aliases are stored lowercased; matching is case-insensitive. +#[derive(Debug)] +struct Entry { + canonical: Box, + /// Lowercased alias spellings. Empty by default. Looked up only + /// after an entry has been chosen as the DL-winner — so aliases + /// listed under entry A never bleed into matches for entry B. + bypass_aliases: HashSet>, +} + /// Domain-specific vocabulary, populated once at server start from /// `*.txt` files under a vocab directory. Read-only thereafter; share /// via `Arc`. #[derive(Default)] pub struct Gazetteer { - /// Canonical entries in original casing — source of truth for both - /// scanning and emission. - entries: Vec>, + /// Canonical entries with their per-entry bypass-alias sets. + entries: Vec, /// Lowercased canonicals; short-circuits replacement when the token /// already matches a known entry exactly. exact: HashSet>, @@ -88,7 +107,10 @@ impl Gazetteer { /// Load every `*.txt` file (non-recursive) under `dir`. Blank lines /// and lines beginning with `#` are skipped. Case-insensitive - /// duplicates across files are silently collapsed. + /// duplicates across files are silently collapsed. A malformed + /// line (e.g. a secondary token without `-` prefix) aborts the load + /// with `io::ErrorKind::InvalidData` — server start fails loud + /// rather than silently misindexing. pub async fn load_dir(dir: &Path) -> io::Result { let mut g = Self::default(); let mut entries = tokio::fs::read_dir(dir).await?; @@ -99,11 +121,15 @@ impl Gazetteer { } let content = tokio::fs::read_to_string(&path).await?; for line in content.lines() { - let trimmed = line.trim(); - if trimmed.is_empty() || trimmed.starts_with('#') { - continue; + let parsed = parse_line(line).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("{}: {e}", path.display()), + ) + })?; + if let Some(entry) = parsed { + g.insert_entry(entry); } - g.insert(trimmed); } } Ok(g) @@ -127,16 +153,38 @@ impl Gazetteer { self } - fn insert(&mut self, entry: &str) { - if entry.chars().count() < MIN_TOKEN_LEN { - return; + /// Insert a bare canonical token with no bypass-aliases. Used by + /// tests and by `insert_entry` as the underlying primitive. + /// Returns false if the token is too short or already indexed. + fn insert_canonical(&mut self, canonical: &str) -> bool { + if canonical.chars().count() < MIN_TOKEN_LEN { + return false; } - let lower = entry.to_lowercase(); + let lower = canonical.to_lowercase(); if !self.exact.insert(lower.into_boxed_str()) { // Duplicate (case-insensitive) — already indexed. + return false; + } + self.entries.push(Entry { + canonical: canonical.into(), + bypass_aliases: HashSet::new(), + }); + true + } + + /// Insert a fully parsed entry. If the canonical is below + /// `MIN_TOKEN_LEN` or duplicates an earlier canonical (case-insensitive), + /// the entry is silently dropped — same dedup behaviour as the + /// original `insert`. Otherwise the bypass-alias set is attached + /// to the new entry. + fn insert_entry(&mut self, entry: Entry) { + if !self.insert_canonical(&entry.canonical) { return; } - self.entries.push(entry.into()); + // Just inserted as the last element; attach its aliases. + if let Some(last) = self.entries.last_mut() { + last.bypass_aliases = entry.bypass_aliases; + } } /// Rewrite every word-token within edit-distance 2 of a gazetteer @@ -181,9 +229,28 @@ impl Gazetteer { let best = self .entries .iter() - .map(|c| (damerau_levenshtein(&lower, &c.to_lowercase()), c.as_ref())) + .map(|e| (damerau_levenshtein(&lower, &e.canonical.to_lowercase()), e)) .filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE) - .min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(b.1)))?; + .min_by(|a, b| { + a.0.cmp(&b.0) + .then_with(|| a.1.canonical.cmp(&b.1.canonical)) + })?; + + let (distance, entry) = best; + + // Per-entry bypass: if the user has explicitly listed `lower` + // as a known mishearing of this canonical, skip the dict-veto. + // Aliases are scoped to the entry that won the DL contest, so + // identical aliases under unrelated entries do not interfere. + if entry.bypass_aliases.contains(lower.as_str()) { + tracing::info!( + token = %token, + canonical = %entry.canonical, + distance, + "gazetteer bypass alias matched", + ); + return Some((entry.canonical.as_ref(), distance)); + } // Dict-veto: only consult the checker *after* we already have // a DL-candidate. Common words (99% of a transcript) never @@ -199,15 +266,15 @@ impl Gazetteer { if dict.check(token) || dict.check(&first_upper) { tracing::info!( token = %token, - candidate = best.1, - distance = best.0, + candidate = %entry.canonical, + distance, "gazetteer blocked by dict veto", ); return None; } } - Some((best.1, best.0)) + Some((entry.canonical.as_ref(), distance)) } } @@ -286,6 +353,79 @@ fn tokenize(text: &str) -> Vec> { out } +/// Errors that can arise while parsing a single vocab-file line. +/// Currently a single variant; kept as an enum so future format +/// extensions stay backwards-compatible with the matchers. +#[derive(Debug)] +pub enum ParseError { + /// A secondary token on a multi-token line did not start with `-`. + /// Example: `"Sellink Seelink"` — the user almost certainly meant + /// `"-Seelink"`. Failing loud at startup beats silent ambiguity. + SecondaryWithoutDash { line: String, token: String }, +} + +impl fmt::Display for ParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::SecondaryWithoutDash { line, token } => write!( + f, + "vocab line {line:?}: secondary token {token:?} must start with '-' \ + to mark it as a dict-veto bypass alias", + ), + } + } +} + +impl std::error::Error for ParseError {} + +/// Parse a single vocab-file line into an `Entry`. +/// +/// Format: +/// - Whitespace-separated tokens. +/// - Blank lines and lines starting with `#` → `Ok(None)`. +/// - First token = canonical. Below `MIN_TOKEN_LEN` → `Ok(None)` +/// (preserves the historical "short entries are silently dropped" +/// behaviour, so vocab files written for the old single-token format +/// keep working unchanged). +/// - Secondary tokens must begin with `-`. The `-` is stripped, the +/// remainder is lowercased and inserted into `bypass_aliases`. A +/// secondary token without `-` is a hard error. +/// - Aliases shorter than `MIN_TOKEN_LEN` are accepted: that floor only +/// exists to protect the DL-heuristic from short-word collisions, but +/// bypass-aliases are exact-match only and explicit, so the floor +/// does not apply. +fn parse_line(line: &str) -> Result, ParseError> { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + return Ok(None); + } + let mut tokens = trimmed.split_whitespace(); + let canonical = match tokens.next() { + Some(c) => c, + None => return Ok(None), + }; + if canonical.chars().count() < MIN_TOKEN_LEN { + return Ok(None); + } + let mut bypass_aliases = HashSet::new(); + for tok in tokens { + let alias = match tok.strip_prefix('-') { + Some(rest) if !rest.is_empty() => rest, + _ => { + return Err(ParseError::SecondaryWithoutDash { + line: trimmed.to_string(), + token: tok.to_string(), + }); + } + }; + bypass_aliases.insert(alias.to_lowercase().into_boxed_str()); + } + Ok(Some(Entry { + canonical: canonical.into(), + bypass_aliases, + })) +} + #[cfg(test)] mod tests { use super::*; @@ -294,7 +434,21 @@ mod tests { fn from_entries(entries: &[&str]) -> Gazetteer { let mut g = Gazetteer::empty(); for e in entries { - g.insert(e); + g.insert_canonical(e); + } + g + } + + /// Build a gazetteer from full vocab-file lines (canonical plus + /// optional `-alias` tokens). Panics on parse error — tests should + /// pass well-formed lines or use `parse_line` directly to assert + /// errors. + fn from_lines(lines: &[&str]) -> Gazetteer { + let mut g = Gazetteer::empty(); + for line in lines { + if let Some(entry) = parse_line(line).expect("parse_line") { + g.insert_entry(entry); + } } g } @@ -519,6 +673,144 @@ mod tests { ); } + // --- Bypass-alias parser tests --- + + /// Backward-compatibility: a single-token line still produces a + /// canonical entry with no aliases. All historical vocab files + /// stay valid under the new parser. + #[test] + fn parse_line_single_token_unchanged() { + let entry = parse_line("Sellink").unwrap().unwrap(); + assert_eq!(&*entry.canonical, "Sellink"); + assert!(entry.bypass_aliases.is_empty()); + } + + /// Canonical plus one bypass-alias. The alias is stored lowercased + /// (matching is case-insensitive); the canonical preserves its + /// original casing for emission. + #[test] + fn parse_line_with_bypass_alias() { + let entry = parse_line("Sellink -Seelink").unwrap().unwrap(); + assert_eq!(&*entry.canonical, "Sellink"); + assert!(entry.bypass_aliases.contains("seelink")); + assert_eq!(entry.bypass_aliases.len(), 1); + } + + /// Multiple aliases on the same line, all dash-prefixed. + #[test] + fn parse_line_multiple_aliases() { + let entry = parse_line("Lantus -Latus -Lattus").unwrap().unwrap(); + assert_eq!(&*entry.canonical, "Lantus"); + assert!(entry.bypass_aliases.contains("latus")); + assert!(entry.bypass_aliases.contains("lattus")); + assert_eq!(entry.bypass_aliases.len(), 2); + } + + /// A secondary token without `-` is a hard parse error. The + /// server-start path converts this to `io::ErrorKind::InvalidData`. + #[test] + fn parse_line_rejects_secondary_without_dash() { + let err = parse_line("Sellink Seelink").unwrap_err(); + match err { + ParseError::SecondaryWithoutDash { line, token } => { + assert_eq!(line, "Sellink Seelink"); + assert_eq!(token, "Seelink"); + } + } + } + + /// Blank lines and `#`-comments produce `Ok(None)`, just like the + /// old loop's `continue`. + #[test] + fn parse_line_skips_blank_and_comment() { + assert!(parse_line("").unwrap().is_none()); + assert!(parse_line(" ").unwrap().is_none()); + assert!(parse_line("# a comment").unwrap().is_none()); + assert!(parse_line(" # indented comment").unwrap().is_none()); + } + + /// A bare `-` (no alias body) is treated as a malformed secondary + /// token, not as an empty alias. + #[test] + fn parse_line_rejects_bare_dash() { + let err = parse_line("Sellink -").unwrap_err(); + assert!(matches!(err, ParseError::SecondaryWithoutDash { .. })); + } + + // --- Bypass-alias behavioural tests --- + + /// Core bug-fix test for the 2026-04-30 log finding. Whisper + /// produced "Seelink", which DL=1 matches "Sellink". Hunspell-de + /// (here mocked) accepts "Seelink" as a valid German compound and + /// would veto the replacement. The bypass-alias forces the rewrite + /// regardless. + #[test] + fn bypass_alias_overrides_dict_veto() { + let g = + from_lines(&["Sellink -Seelink"]).with_dict(Box::new(HashSetDict::new(&["Seelink"]))); + assert_eq!(g.replace("MRT-Seelink heute."), "MRT-Sellink heute."); + } + + /// Bypass-aliases are exact-match only (case-insensitive). A + /// declension like "Seelinks" is not in the alias set, so it falls + /// through to the normal DL+veto path. Here the DL=2 candidate + /// would be vetoed by the dict (which "knows" the plural), so the + /// token stays untouched — proving the bypass does not bleed. + #[test] + fn bypass_alias_is_exact_match_only() { + let g = + from_lines(&["Sellink -Seelink"]).with_dict(Box::new(HashSetDict::new(&["Seelinks"]))); + assert_eq!(g.replace("Sieht Seelinks aus."), "Sieht Seelinks aus."); + } + + /// An alias listed under entry A must not enable a bypass for + /// entry B's matches. Here only `Sellink` has the bypass, but the + /// token under test (`Kaktus`) is DL=2 from `Lantus`. The dict + /// veto must still block the Kaktus → Lantus rewrite. + #[test] + fn bypass_alias_does_not_affect_other_entries() { + let g = from_lines(&["Sellink -Seelink", "Lantus"]) + .with_dict(Box::new(HashSetDict::new(&["Kaktus"]))); + assert_eq!( + g.replace("Patient hat einen Kaktus."), + "Patient hat einen Kaktus." + ); + } + + /// Case-insensitive alias matching: the alias is stored lowercased, + /// so any casing of the input token that lowercases to the alias + /// triggers the bypass. + #[test] + fn bypass_alias_is_case_insensitive() { + let g = from_lines(&["Sellink -Seelink"]).with_dict(Box::new(HashSetDict::new(&[ + "Seelink", "seelink", "SEELINK", + ]))); + assert_eq!(g.replace("Mit SEELINK heute."), "Mit Sellink heute."); + assert_eq!(g.replace("mit seelink heute."), "mit Sellink heute."); + } + + /// `load_dir` propagates a parser error as `InvalidData`, with the + /// offending file path in the message — server-start should fail + /// loud rather than silently misindex. + #[tokio::test] + async fn load_dir_propagates_parse_error_as_invalid_data() { + let dir = tempdir().unwrap(); + tokio::fs::write(dir.path().join("v.txt"), "Sellink Seelink\n") + .await + .unwrap(); + let err = Gazetteer::load_dir(dir.path()).await.unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + let msg = err.to_string(); + assert!( + msg.contains("Seelink"), + "error message must name the bad token: {msg}" + ); + assert!( + msg.contains("v.txt"), + "error message must name the file: {msg}" + ); + } + /// Live check against the system-installed hunspell-de. /// Marked `#[ignore]` because it depends on /// `/usr/share/hunspell/de_DE.{aff,dic}` being present — the diff --git a/server/vocab/README.md b/server/vocab/README.md index b4e2d51..4c28c55 100644 --- a/server/vocab/README.md +++ b/server/vocab/README.md @@ -14,6 +14,31 @@ Whisper transcripts as `Token [?Canonical]` for LLM-side review. everyday German words). - Duplicates (case-insensitive, across files) are collapsed automatically. +### Optional: bypass-aliases for false-positive Hunspell compounds + +A line may carry one or more `-`-prefixed alias tokens after the canonical: + +``` +Sellink -Seelink -Seelinks +``` + +Meaning: when a transcript token (case-insensitive) exactly matches a +listed alias, the gazetteer skips the Hunspell dict-veto and rewrites +the token to the canonical. Use this **only** when Hunspell-de +mistakenly accepts a known mishearing as a valid German compound and +thereby blocks a legitimate correction (e.g. Hunspell parses `Seelink` +as `See` + `link` and vetoes `Seelink → Sellink`). + +- The `-` is the bypass marker; it is stripped before storage. +- Aliases are **exact match only** (no Damerau-Levenshtein tolerance). + If a new declension or mishearing surfaces, append it explicitly: + `Sellink -Seelink -Seelinks`. +- Aliases are scoped to the entry whose canonical wins the DL contest; + identical aliases under unrelated entries do not interfere. +- A secondary token without `-` is a hard server-start error + (`io::ErrorKind::InvalidData`) — the loader fails loud rather than + silently misindex. + ## Adding entries Edit `vocabulary.txt` (or drop additional `*.txt` files into this diff --git a/server/vocab/vocabulary.txt b/server/vocab/vocabulary.txt index 4d53d79..0ac17b6 100644 --- a/server/vocab/vocabulary.txt +++ b/server/vocab/vocabulary.txt @@ -127,7 +127,7 @@ Risperidon Rivaroxaban Rosuvastatin Salbutamol -Sellink +Sellink -Seelink Sertralin Sildenafil Simvastatin