diff --git a/server/src/gazetteer/mod.rs b/server/src/gazetteer/mod.rs index fa86db1..2f6d990 100644 --- a/server/src/gazetteer/mod.rs +++ b/server/src/gazetteer/mod.rs @@ -3,10 +3,12 @@ //! persistence. //! //! Invariant: every AI-produced text passes through `replace()` before -//! being written to disk or forwarded downstream. Tokens within -//! edit-distance 2 of a curated vocabulary entry are silently rewritten -//! to the canonical form — *unless* the token itself is a valid word -//! of the target language (dict-veto), in which case it is preserved. +//! being written to disk or forwarded downstream. Tokens within the +//! per-entry edit-distance threshold (default 2, configurable up to +//! `HARD_MAX_EDIT_DISTANCE` via the `!N` annotation in the vocab file) +//! are silently rewritten to the canonical form — *unless* the token +//! itself is a valid word of the target language (dict-veto), in +//! which case it is preserved. //! //! Rationale: LLMs have strong training priors that override contextual //! hints — e.g. they anglicize `Amiodaron` to `Amiodarone` even when @@ -16,9 +18,11 @@ //! mechanism. //! //! Matcher: single-stage linear Damerau-Levenshtein scan over all -//! entries, filtered by distance ≤ MAX_EDIT_DISTANCE. At typical sizes -//! (~200 entries × ~50 tokens), the per-pass cost stays well under a -//! millisecond — negligible next to the AI call it follows. +//! entries, filtered by each entry's `max_distance`. A length-difference +//! pre-filter short-circuits the O(len_a × len_b) DL computation when +//! the absolute length gap already exceeds the threshold. At typical +//! sizes (~200 entries × ~50 tokens), the per-pass cost stays well +//! under a millisecond — negligible next to the AI call it follows. //! //! Dict-veto: if a token is within the edit-distance window of a //! vocab entry but is itself a valid word of the language (checked @@ -41,10 +45,18 @@ use strsim::damerau_levenshtein; /// (Bein↔Behn, nach↔Nuck). const MIN_TOKEN_LEN: usize = 5; -/// Maximum Damerau-Levenshtein distance for a candidate to qualify as a -/// typo of a gazetteer entry. Distance 0 is handled by the `exact` -/// short-circuit; distances > 2 are rejected as too speculative. -const MAX_EDIT_DISTANCE: usize = 2; +/// Default Damerau-Levenshtein threshold applied to every entry that +/// does not carry a per-entry override. Distance 0 is handled by the +/// `exact` short-circuit; distances above the entry's threshold are +/// rejected as too speculative. +const DEFAULT_MAX_EDIT_DISTANCE: usize = 2; + +/// Hard upper bound for any per-entry `!N` override the parser will +/// accept. Beyond DL=3 on typical 8-char vocabulary entries we are no +/// longer correcting mishearings; we are inventing words. The cap +/// keeps curators honest — if you find yourself needing `!4`, the +/// right next step is a phonetic stage, not a wider edit window. +const HARD_MAX_EDIT_DISTANCE: usize = 3; /// Veto gate for `replace()`: if `check()` returns true for a token, /// the token is considered a valid word of the language and is @@ -69,10 +81,28 @@ pub trait DictChecker: Send + Sync { #[derive(Debug)] struct Entry { canonical: Box, + /// Lowercased canonical, cached at load time. Used in the DL hot + /// path so we don't allocate a fresh `String` per token-comparison. + /// At ~200 entries × 50 tokens that saves 10k allocations per + /// `replace()` call. + canonical_lower: Box, + /// Cached character count of `canonical_lower`. Feeds the + /// length-difference pre-filter that short-circuits the DL + /// computation when |len_a - len_b| already exceeds `max_distance`. + /// `chars().count()` is O(n); caching it once at load time turns + /// the hot-path check into a single `usize::abs_diff`. + canonical_chars_len: usize, /// Lowercased alias spellings. Empty by default. Looked up only /// after an entry has been chosen as the DL-winner — so aliases /// listed under entry A never bleed into matches for entry B. bypass_aliases: HashSet>, + /// Per-entry edit-distance threshold. Defaults to + /// `DEFAULT_MAX_EDIT_DISTANCE`; overridden via the `!N` token in + /// the vocab file (validated against `HARD_MAX_EDIT_DISTANCE`). + /// Living on the entry rather than as a global keeps the configuration + /// local — readers can see *why* a given vocabulary line tolerates + /// `!3` without hunting a global switch. + max_distance: usize, } /// Domain-specific vocabulary, populated once at server start from @@ -160,14 +190,18 @@ impl Gazetteer { if canonical.chars().count() < MIN_TOKEN_LEN { return false; } - let lower = canonical.to_lowercase(); - if !self.exact.insert(lower.into_boxed_str()) { + let canonical_lower: Box = canonical.to_lowercase().into_boxed_str(); + let canonical_chars_len = canonical_lower.chars().count(); + if !self.exact.insert(canonical_lower.clone()) { // Duplicate (case-insensitive) — already indexed. return false; } self.entries.push(Entry { canonical: canonical.into(), + canonical_lower, + canonical_chars_len, bypass_aliases: HashSet::new(), + max_distance: DEFAULT_MAX_EDIT_DISTANCE, }); true } @@ -181,9 +215,11 @@ impl Gazetteer { if !self.insert_canonical(&entry.canonical) { return; } - // Just inserted as the last element; attach its aliases. + // Just inserted as the last element; attach its per-entry + // configuration (aliases + distance override). if let Some(last) = self.entries.last_mut() { last.bypass_aliases = entry.bypass_aliases; + last.max_distance = entry.max_distance; } } @@ -218,7 +254,8 @@ impl Gazetteer { } fn find_candidate(&self, token: &str) -> Option<(&str, usize)> { - if token.chars().count() < MIN_TOKEN_LEN { + let token_chars_len = token.chars().count(); + if token_chars_len < MIN_TOKEN_LEN { return None; } let lower = token.to_lowercase(); @@ -226,11 +263,21 @@ impl Gazetteer { return None; } + // Length pre-filter: DL is a metric, so DL(a, b) >= ||a| - |b||. + // If the length gap already exceeds the entry's threshold, the + // O(len_a × len_b) DL computation cannot possibly succeed — + // skip it. At ~200 entries this pruning is small; once the + // vocabulary grows it becomes the dominant savings. let best = self .entries .iter() - .map(|e| (damerau_levenshtein(&lower, &e.canonical.to_lowercase()), e)) - .filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE) + .filter_map(|e| { + if token_chars_len.abs_diff(e.canonical_chars_len) > e.max_distance { + return None; + } + let d = damerau_levenshtein(&lower, &e.canonical_lower); + (d >= 1 && d <= e.max_distance).then_some((d, e)) + }) .min_by(|a, b| { a.0.cmp(&b.0) .then_with(|| a.1.canonical.cmp(&b.1.canonical)) @@ -354,23 +401,47 @@ fn tokenize(text: &str) -> Vec> { } /// Errors that can arise while parsing a single vocab-file line. -/// Currently a single variant; kept as an enum so future format -/// extensions stay backwards-compatible with the matchers. #[derive(Debug)] pub enum ParseError { - /// A secondary token on a multi-token line did not start with `-`. - /// Example: `"Sellink Seelink"` — the user almost certainly meant - /// `"-Seelink"`. Failing loud at startup beats silent ambiguity. - SecondaryWithoutDash { line: String, token: String }, + /// A secondary token on a multi-token line did not start with `-` + /// (alias) or `!` (distance override). Example: `"Sellink Seelink"` — + /// the user almost certainly meant `"-Seelink"`. Failing loud at + /// startup beats silent ambiguity. + UnknownSecondaryToken { line: String, token: String }, + /// A `!N` token whose `N` part did not parse as a non-negative + /// integer. Example: `"Foo !abc"`. + InvalidDistance { line: String, token: String }, + /// A `!N` token whose value lies outside `1..=HARD_MAX_EDIT_DISTANCE`. + /// `!0` would be useless (DL=0 is the exact-match short-circuit); + /// values above the hard cap risk widespread false-positives across + /// the vocabulary and are refused by design. + DistanceOutOfRange { line: String, value: usize }, + /// More than one `!N` token on the same line. Ambiguous intent; + /// reject rather than guess which override "wins". + DuplicateDistanceOverride { line: String }, } impl fmt::Display for ParseError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::SecondaryWithoutDash { line, token } => write!( + Self::UnknownSecondaryToken { line, token } => write!( f, "vocab line {line:?}: secondary token {token:?} must start with '-' \ - to mark it as a dict-veto bypass alias", + (bypass alias) or '!' (distance override)", + ), + Self::InvalidDistance { line, token } => write!( + f, + "vocab line {line:?}: distance override {token:?} must be of the form \ + !N where N is a non-negative integer", + ), + Self::DistanceOutOfRange { line, value } => write!( + f, + "vocab line {line:?}: distance override !{value} is out of range \ + (allowed: 1..={HARD_MAX_EDIT_DISTANCE})", + ), + Self::DuplicateDistanceOverride { line } => write!( + f, + "vocab line {line:?}: more than one distance override (!N) on the same line", ), } } @@ -387,9 +458,12 @@ impl std::error::Error for ParseError {} /// (preserves the historical "short entries are silently dropped" /// behaviour, so vocab files written for the old single-token format /// keep working unchanged). -/// - Secondary tokens must begin with `-`. The `-` is stripped, the -/// remainder is lowercased and inserted into `bypass_aliases`. A -/// secondary token without `-` is a hard error. +/// - Secondary tokens are prefixed: +/// - `-alias` → bypass-alias (lowercased, exact-match veto-override). +/// - `!N` → per-entry max edit distance (N in `1..=HARD_MAX_EDIT_DISTANCE`). +/// +/// Any other secondary form is a hard parse error. +/// - At most one `!N` per line; a second one is rejected as ambiguous. /// - Aliases shorter than `MIN_TOKEN_LEN` are accepted: that floor only /// exists to protect the DL-heuristic from short-word collisions, but /// bypass-aliases are exact-match only and explicit, so the floor @@ -408,21 +482,48 @@ fn parse_line(line: &str) -> Result, ParseError> { return Ok(None); } let mut bypass_aliases = HashSet::new(); + let mut max_distance: Option = None; for tok in tokens { - let alias = match tok.strip_prefix('-') { - Some(rest) if !rest.is_empty() => rest, - _ => { - return Err(ParseError::SecondaryWithoutDash { + if let Some(rest) = tok.strip_prefix('-') { + if rest.is_empty() { + return Err(ParseError::UnknownSecondaryToken { line: trimmed.to_string(), token: tok.to_string(), }); } - }; - bypass_aliases.insert(alias.to_lowercase().into_boxed_str()); + bypass_aliases.insert(rest.to_lowercase().into_boxed_str()); + } else if let Some(rest) = tok.strip_prefix('!') { + let value: usize = rest.parse().map_err(|_| ParseError::InvalidDistance { + line: trimmed.to_string(), + token: tok.to_string(), + })?; + if !(1..=HARD_MAX_EDIT_DISTANCE).contains(&value) { + return Err(ParseError::DistanceOutOfRange { + line: trimmed.to_string(), + value, + }); + } + if max_distance.is_some() { + return Err(ParseError::DuplicateDistanceOverride { + line: trimmed.to_string(), + }); + } + max_distance = Some(value); + } else { + return Err(ParseError::UnknownSecondaryToken { + line: trimmed.to_string(), + token: tok.to_string(), + }); + } } + let canonical_lower: Box = canonical.to_lowercase().into_boxed_str(); + let canonical_chars_len = canonical_lower.chars().count(); Ok(Some(Entry { canonical: canonical.into(), + canonical_lower, + canonical_chars_len, bypass_aliases, + max_distance: max_distance.unwrap_or(DEFAULT_MAX_EDIT_DISTANCE), })) } @@ -706,16 +807,17 @@ mod tests { assert_eq!(entry.bypass_aliases.len(), 2); } - /// A secondary token without `-` is a hard parse error. The + /// A secondary token without `-` or `!` is a hard parse error. The /// server-start path converts this to `io::ErrorKind::InvalidData`. #[test] - fn parse_line_rejects_secondary_without_dash() { + fn parse_line_rejects_secondary_without_prefix() { let err = parse_line("Sellink Seelink").unwrap_err(); match err { - ParseError::SecondaryWithoutDash { line, token } => { + ParseError::UnknownSecondaryToken { line, token } => { assert_eq!(line, "Sellink Seelink"); assert_eq!(token, "Seelink"); } + other => panic!("expected UnknownSecondaryToken, got {other:?}"), } } @@ -734,7 +836,7 @@ mod tests { #[test] fn parse_line_rejects_bare_dash() { let err = parse_line("Sellink -").unwrap_err(); - assert!(matches!(err, ParseError::SecondaryWithoutDash { .. })); + assert!(matches!(err, ParseError::UnknownSecondaryToken { .. })); } // --- Bypass-alias behavioural tests --- @@ -789,6 +891,179 @@ mod tests { assert_eq!(g.replace("mit seelink heute."), "mit Sellink heute."); } + // --- Per-entry distance override (!N) parser tests --- + + /// Default behaviour preserved: a single-token line yields + /// `max_distance = DEFAULT_MAX_EDIT_DISTANCE`. + #[test] + fn parse_line_default_distance_is_two() { + let entry = parse_line("Aspirin").unwrap().unwrap(); + assert_eq!(entry.max_distance, DEFAULT_MAX_EDIT_DISTANCE); + assert_eq!(entry.max_distance, 2); + } + + /// Loosen: `!3` opens an entry to DL=3 — the use case that + /// motivated this whole feature (Thorazemit ↔ Torasemid). + #[test] + fn parse_line_with_distance_override_three() { + let entry = parse_line("Torasemid !3").unwrap().unwrap(); + assert_eq!(&*entry.canonical, "Torasemid"); + assert_eq!(entry.max_distance, 3); + assert!(entry.bypass_aliases.is_empty()); + } + + /// Tighten: `!1` shrinks an entry to DL=1 — useful for short + /// canonicals where DL=2 admits too many neighbours. + #[test] + fn parse_line_accepts_distance_one() { + let entry = parse_line("Aspirin !1").unwrap().unwrap(); + assert_eq!(entry.max_distance, 1); + } + + /// `!N` and `-alias` are independent and may coexist on the same + /// line. Order does not matter to the parser. + #[test] + fn parse_line_combines_distance_and_alias() { + let entry = parse_line("Torasemid !3 -Thorasemid").unwrap().unwrap(); + assert_eq!(&*entry.canonical, "Torasemid"); + assert_eq!(entry.max_distance, 3); + assert!(entry.bypass_aliases.contains("thorasemid")); + } + + /// `!N` may appear before or after `-alias` tokens — whitespace + /// order is irrelevant. Regression guard against accidentally + /// coupling the parser to token order. + #[test] + fn parse_line_alias_then_distance_works() { + let entry = parse_line("Torasemid -Thorasemid !3").unwrap().unwrap(); + assert_eq!(entry.max_distance, 3); + assert!(entry.bypass_aliases.contains("thorasemid")); + } + + /// Above the hard cap (`HARD_MAX_EDIT_DISTANCE = 3`) is rejected. + /// The cap is intentional: edit-distance 4 on typical 8-char + /// vocabulary entries flips half the word and is no longer a + /// mishearing — extending here is the wrong fix. + #[test] + fn parse_line_rejects_distance_above_cap() { + let err = parse_line("Aspirin !4").unwrap_err(); + match err { + ParseError::DistanceOutOfRange { value, .. } => assert_eq!(value, 4), + other => panic!("expected DistanceOutOfRange, got {other:?}"), + } + } + + /// `!0` is rejected — DL=0 is the exact-match short-circuit, so + /// an entry tolerating "no edits" would silently never match. + /// Better to fail loud than to ship a useless line. + #[test] + fn parse_line_rejects_distance_zero() { + let err = parse_line("Aspirin !0").unwrap_err(); + assert!(matches!( + err, + ParseError::DistanceOutOfRange { value: 0, .. } + )); + } + + /// Non-numeric `!N` fails with `InvalidDistance`, not silently. + #[test] + fn parse_line_rejects_invalid_distance_token() { + let err = parse_line("Aspirin !abc").unwrap_err(); + match err { + ParseError::InvalidDistance { token, .. } => assert_eq!(token, "!abc"), + other => panic!("expected InvalidDistance, got {other:?}"), + } + } + + /// More than one `!N` on the same line is ambiguous — reject + /// rather than guess which override "wins". + #[test] + fn parse_line_rejects_duplicate_distance() { + let err = parse_line("Aspirin !2 !3").unwrap_err(); + assert!(matches!(err, ParseError::DuplicateDistanceOverride { .. })); + } + + // --- Per-entry distance override (!N) behavioural tests --- + + /// Without the override, a DL=3 token does NOT match — that is + /// the original conservative behaviour and must be preserved. + #[test] + fn find_candidate_dl3_no_match_without_override() { + let g = from_lines(&["Torasemid"]); + // "Thorazemit" is DL=3 from "Torasemid": + // T-orasemid → Thorasemid (insert h) + // s → z (substitution) + // d → t (substitution) + assert_eq!( + g.replace("Patient nimmt Thorazemit heute."), + "Patient nimmt Thorazemit heute." + ); + } + + /// With `!3`, the same DL=3 token now resolves to the canonical. + /// This is the headline behaviour the feature exists for. + #[test] + fn find_candidate_dl3_match_with_per_entry_override() { + let g = from_lines(&["Torasemid !3"]); + assert_eq!( + g.replace("Patient nimmt Thorazemit heute."), + "Patient nimmt Torasemid heute." + ); + } + + /// DL=4 is never matched, even with `!3` — the hard cap holds. + /// Picking a token genuinely 4 edits away from "Torasemid": + /// "Thorazemett" — insert h, s→z, d→t, append t. + #[test] + fn find_candidate_dl4_never_matches_with_override_three() { + let g = from_lines(&["Torasemid !3"]); + assert_eq!( + g.replace("Wert von Thorazemett heute."), + "Wert von Thorazemett heute." + ); + } + + /// `!1` tightens the threshold: a DL=2 token that *would* have + /// matched at the default no longer does. Verifies that the + /// per-entry value really overrides the default in both directions. + #[test] + fn find_candidate_distance_one_tightens_filter() { + // Default would catch DL=2 typo of "Cerebrum"; !1 must not. + let g = from_lines(&["Cerebrum !1"]); + // "Zerabrum": z→c (1) + a→e (1) = DL=2 from "Cerebrum". + assert_eq!(g.replace("Blutung im Zerabrum."), "Blutung im Zerabrum."); + // DL=1 still matches under !1. + assert_eq!(g.replace("Blutung im Zerebrum."), "Blutung im Cerebrum."); + } + + /// `!3` does NOT bypass the dict-veto. If a DL=3 candidate is + /// itself a real word of the language, Hunspell's veto must still + /// hold — the per-entry threshold widens the DL filter, nothing else. + #[test] + fn find_candidate_dl3_blocked_by_dict_veto() { + // Mock dict claims the DL=3 token is a real word. Even with + // !3 admitting it past the DL filter, the veto must keep it. + let g = + from_lines(&["Torasemid !3"]).with_dict(Box::new(HashSetDict::new(&["Thorazemit"]))); + assert_eq!( + g.replace("Patient nimmt Thorazemit heute."), + "Patient nimmt Thorazemit heute." + ); + } + + /// Bypass-aliases compose with `!3`: an explicit alias still + /// overrides the dict-veto on a !3-entry, just as it does on a + /// default-distance entry. + #[test] + fn bypass_alias_works_alongside_distance_override() { + let g = from_lines(&["Torasemid !3 -Thorazemit"]) + .with_dict(Box::new(HashSetDict::new(&["Thorazemit"]))); + assert_eq!( + g.replace("Patient nimmt Thorazemit heute."), + "Patient nimmt Torasemid heute." + ); + } + /// `load_dir` propagates a parser error as `InvalidData`, with the /// offending file path in the message — server-start should fail /// loud rather than silently misindex. diff --git a/server/vocab/vocabulary.txt b/server/vocab/vocabulary.txt index ebcabc2..4cbe22c 100644 --- a/server/vocab/vocabulary.txt +++ b/server/vocab/vocabulary.txt @@ -146,7 +146,7 @@ Telmisartan Temesta Thyroxin Timolol -Torasemid +Torasemid !3 Tramadol Tranxilium Trulicity