Refactor gazetteer to support per-entry edit distance

server/src/gazetteer/mod.rs

@@ -3,10 +3,12 @@
 //! persistence.
 //!
 //! Invariant: every AI-produced text passes through `replace()` before
-//! being written to disk or forwarded downstream. Tokens within
+//! being written to disk or forwarded downstream. Tokens within the
-//! edit-distance 2 of a curated vocabulary entry are silently rewritten
+//! per-entry edit-distance threshold (default 2, configurable up to
-//! to the canonical form — *unless* the token itself is a valid word
+//! `HARD_MAX_EDIT_DISTANCE` via the `!N` annotation in the vocab file)
-//! of the target language (dict-veto), in which case it is preserved.
+//! are silently rewritten to the canonical form — *unless* the token
+//! itself is a valid word of the target language (dict-veto), in
+//! which case it is preserved.
 //!
 //! Rationale: LLMs have strong training priors that override contextual
 //! hints — e.g. they anglicize `Amiodaron` to `Amiodarone` even when
@@ -16,9 +18,11 @@
 //! mechanism.
 //!
 //! Matcher: single-stage linear Damerau-Levenshtein scan over all
-//! entries, filtered by distance ≤ MAX_EDIT_DISTANCE. At typical sizes
+//! entries, filtered by each entry's `max_distance`. A length-difference
-//! (~200 entries × ~50 tokens), the per-pass cost stays well under a
+//! pre-filter short-circuits the O(len_a × len_b) DL computation when
-//! millisecond — negligible next to the AI call it follows.
+//! the absolute length gap already exceeds the threshold. At typical
+//! sizes (~200 entries × ~50 tokens), the per-pass cost stays well
+//! under a millisecond — negligible next to the AI call it follows.
 //!
 //! Dict-veto: if a token is within the edit-distance window of a
 //! vocab entry but is itself a valid word of the language (checked
@@ -41,10 +45,18 @@ use strsim::damerau_levenshtein;
 /// (Bein↔Behn, nach↔Nuck).
 const MIN_TOKEN_LEN: usize = 5;
 
-/// Maximum Damerau-Levenshtein distance for a candidate to qualify as a
+/// Default Damerau-Levenshtein threshold applied to every entry that
-/// typo of a gazetteer entry. Distance 0 is handled by the `exact`
+/// does not carry a per-entry override. Distance 0 is handled by the
-/// short-circuit; distances > 2 are rejected as too speculative.
+/// `exact` short-circuit; distances above the entry's threshold are
-const MAX_EDIT_DISTANCE: usize = 2;
+/// rejected as too speculative.
+const DEFAULT_MAX_EDIT_DISTANCE: usize = 2;
+
+/// Hard upper bound for any per-entry `!N` override the parser will
+/// accept. Beyond DL=3 on typical 8-char vocabulary entries we are no
+/// longer correcting mishearings; we are inventing words. The cap
+/// keeps curators honest — if you find yourself needing `!4`, the
+/// right next step is a phonetic stage, not a wider edit window.
+const HARD_MAX_EDIT_DISTANCE: usize = 3;
 
 /// Veto gate for `replace()`: if `check()` returns true for a token,
 /// the token is considered a valid word of the language and is
@@ -69,10 +81,28 @@ pub trait DictChecker: Send + Sync {
 #[derive(Debug)]
 struct Entry {
     canonical: Box<str>,
+    /// Lowercased canonical, cached at load time. Used in the DL hot
+    /// path so we don't allocate a fresh `String` per token-comparison.
+    /// At ~200 entries × 50 tokens that saves 10k allocations per
+    /// `replace()` call.
+    canonical_lower: Box<str>,
+    /// Cached character count of `canonical_lower`. Feeds the
+    /// length-difference pre-filter that short-circuits the DL
+    /// computation when |len_a - len_b| already exceeds `max_distance`.
+    /// `chars().count()` is O(n); caching it once at load time turns
+    /// the hot-path check into a single `usize::abs_diff`.
+    canonical_chars_len: usize,
     /// Lowercased alias spellings. Empty by default. Looked up only
     /// after an entry has been chosen as the DL-winner — so aliases
     /// listed under entry A never bleed into matches for entry B.
     bypass_aliases: HashSet<Box<str>>,
+    /// Per-entry edit-distance threshold. Defaults to
+    /// `DEFAULT_MAX_EDIT_DISTANCE`; overridden via the `!N` token in
+    /// the vocab file (validated against `HARD_MAX_EDIT_DISTANCE`).
+    /// Living on the entry rather than as a global keeps the configuration
+    /// local — readers can see *why* a given vocabulary line tolerates
+    /// `!3` without hunting a global switch.
+    max_distance: usize,
 }
 
 /// Domain-specific vocabulary, populated once at server start from
@@ -160,14 +190,18 @@ impl Gazetteer {
         if canonical.chars().count() < MIN_TOKEN_LEN {
             return false;
         }
-        let lower = canonical.to_lowercase();
+        let canonical_lower: Box<str> = canonical.to_lowercase().into_boxed_str();
-        if !self.exact.insert(lower.into_boxed_str()) {
+        let canonical_chars_len = canonical_lower.chars().count();
+        if !self.exact.insert(canonical_lower.clone()) {
             // Duplicate (case-insensitive) — already indexed.
             return false;
         }
         self.entries.push(Entry {
             canonical: canonical.into(),
+            canonical_lower,
+            canonical_chars_len,
             bypass_aliases: HashSet::new(),
+            max_distance: DEFAULT_MAX_EDIT_DISTANCE,
         });
         true
     }
@@ -181,9 +215,11 @@ impl Gazetteer {
         if !self.insert_canonical(&entry.canonical) {
             return;
         }
-        // Just inserted as the last element; attach its aliases.
+        // Just inserted as the last element; attach its per-entry
+        // configuration (aliases + distance override).
         if let Some(last) = self.entries.last_mut() {
             last.bypass_aliases = entry.bypass_aliases;
+            last.max_distance = entry.max_distance;
         }
     }
 
@@ -218,7 +254,8 @@ impl Gazetteer {
     }
 
     fn find_candidate(&self, token: &str) -> Option<(&str, usize)> {
-        if token.chars().count() < MIN_TOKEN_LEN {
+        let token_chars_len = token.chars().count();
+        if token_chars_len < MIN_TOKEN_LEN {
             return None;
         }
         let lower = token.to_lowercase();
@@ -226,11 +263,21 @@ impl Gazetteer {
             return None;
         }
 
+        // Length pre-filter: DL is a metric, so DL(a, b) >= ||a| - |b||.
+        // If the length gap already exceeds the entry's threshold, the
+        // O(len_a × len_b) DL computation cannot possibly succeed —
+        // skip it. At ~200 entries this pruning is small; once the
+        // vocabulary grows it becomes the dominant savings.
         let best = self
             .entries
             .iter()
-            .map(|e| (damerau_levenshtein(&lower, &e.canonical.to_lowercase()), e))
+            .filter_map(|e| {
-            .filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE)
+                if token_chars_len.abs_diff(e.canonical_chars_len) > e.max_distance {
+                    return None;
+                }
+                let d = damerau_levenshtein(&lower, &e.canonical_lower);
+                (d >= 1 && d <= e.max_distance).then_some((d, e))
+            })
             .min_by(|a, b| {
                 a.0.cmp(&b.0)
                     .then_with(|| a.1.canonical.cmp(&b.1.canonical))
@@ -354,23 +401,47 @@ fn tokenize(text: &str) -> Vec<Segment<'_>> {
 }
 
 /// Errors that can arise while parsing a single vocab-file line.
-/// Currently a single variant; kept as an enum so future format
-/// extensions stay backwards-compatible with the matchers.
 #[derive(Debug)]
 pub enum ParseError {
-    /// A secondary token on a multi-token line did not start with `-`.
+    /// A secondary token on a multi-token line did not start with `-`
-    /// Example: `"Sellink Seelink"` — the user almost certainly meant
+    /// (alias) or `!` (distance override). Example: `"Sellink Seelink"` —
-    /// `"-Seelink"`. Failing loud at startup beats silent ambiguity.
+    /// the user almost certainly meant `"-Seelink"`. Failing loud at
-    SecondaryWithoutDash { line: String, token: String },
+    /// startup beats silent ambiguity.
+    UnknownSecondaryToken { line: String, token: String },
+    /// A `!N` token whose `N` part did not parse as a non-negative
+    /// integer. Example: `"Foo !abc"`.
+    InvalidDistance { line: String, token: String },
+    /// A `!N` token whose value lies outside `1..=HARD_MAX_EDIT_DISTANCE`.
+    /// `!0` would be useless (DL=0 is the exact-match short-circuit);
+    /// values above the hard cap risk widespread false-positives across
+    /// the vocabulary and are refused by design.
+    DistanceOutOfRange { line: String, value: usize },
+    /// More than one `!N` token on the same line. Ambiguous intent;
+    /// reject rather than guess which override "wins".
+    DuplicateDistanceOverride { line: String },
 }
 
 impl fmt::Display for ParseError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
-            Self::SecondaryWithoutDash { line, token } => write!(
+            Self::UnknownSecondaryToken { line, token } => write!(
                 f,
                 "vocab line {line:?}: secondary token {token:?} must start with '-' \
-                 to mark it as a dict-veto bypass alias",
+                 (bypass alias) or '!' (distance override)",
+            ),
+            Self::InvalidDistance { line, token } => write!(
+                f,
+                "vocab line {line:?}: distance override {token:?} must be of the form \
+                 !N where N is a non-negative integer",
+            ),
+            Self::DistanceOutOfRange { line, value } => write!(
+                f,
+                "vocab line {line:?}: distance override !{value} is out of range \
+                 (allowed: 1..={HARD_MAX_EDIT_DISTANCE})",
+            ),
+            Self::DuplicateDistanceOverride { line } => write!(
+                f,
+                "vocab line {line:?}: more than one distance override (!N) on the same line",
             ),
         }
     }
@@ -387,9 +458,12 @@ impl std::error::Error for ParseError {}
 ///   (preserves the historical "short entries are silently dropped"
 ///   behaviour, so vocab files written for the old single-token format
 ///   keep working unchanged).
-/// - Secondary tokens must begin with `-`. The `-` is stripped, the
+/// - Secondary tokens are prefixed:
-///   remainder is lowercased and inserted into `bypass_aliases`. A
+///   - `-alias` → bypass-alias (lowercased, exact-match veto-override).
-///   secondary token without `-` is a hard error.
+///   - `!N` → per-entry max edit distance (N in `1..=HARD_MAX_EDIT_DISTANCE`).
+///
+///   Any other secondary form is a hard parse error.
+/// - At most one `!N` per line; a second one is rejected as ambiguous.
 /// - Aliases shorter than `MIN_TOKEN_LEN` are accepted: that floor only
 ///   exists to protect the DL-heuristic from short-word collisions, but
 ///   bypass-aliases are exact-match only and explicit, so the floor
@@ -408,21 +482,48 @@ fn parse_line(line: &str) -> Result<Option<Entry>, ParseError> {
         return Ok(None);
     }
     let mut bypass_aliases = HashSet::new();
+    let mut max_distance: Option<usize> = None;
     for tok in tokens {
-        let alias = match tok.strip_prefix('-') {
+        if let Some(rest) = tok.strip_prefix('-') {
-            Some(rest) if !rest.is_empty() => rest,
+            if rest.is_empty() {
-            _ => {
+                return Err(ParseError::UnknownSecondaryToken {
-                return Err(ParseError::SecondaryWithoutDash {
                     line: trimmed.to_string(),
                     token: tok.to_string(),
                 });
             }
-        };
+            bypass_aliases.insert(rest.to_lowercase().into_boxed_str());
-        bypass_aliases.insert(alias.to_lowercase().into_boxed_str());
+        } else if let Some(rest) = tok.strip_prefix('!') {
+            let value: usize = rest.parse().map_err(|_| ParseError::InvalidDistance {
+                line: trimmed.to_string(),
+                token: tok.to_string(),
+            })?;
+            if !(1..=HARD_MAX_EDIT_DISTANCE).contains(&value) {
+                return Err(ParseError::DistanceOutOfRange {
+                    line: trimmed.to_string(),
+                    value,
+                });
+            }
+            if max_distance.is_some() {
+                return Err(ParseError::DuplicateDistanceOverride {
+                    line: trimmed.to_string(),
+                });
+            }
+            max_distance = Some(value);
+        } else {
+            return Err(ParseError::UnknownSecondaryToken {
+                line: trimmed.to_string(),
+                token: tok.to_string(),
+            });
+        }
     }
+    let canonical_lower: Box<str> = canonical.to_lowercase().into_boxed_str();
+    let canonical_chars_len = canonical_lower.chars().count();
     Ok(Some(Entry {
         canonical: canonical.into(),
+        canonical_lower,
+        canonical_chars_len,
         bypass_aliases,
+        max_distance: max_distance.unwrap_or(DEFAULT_MAX_EDIT_DISTANCE),
     }))
 }
 
@@ -706,16 +807,17 @@ mod tests {
         assert_eq!(entry.bypass_aliases.len(), 2);
     }
 
-    /// A secondary token without `-` is a hard parse error. The
+    /// A secondary token without `-` or `!` is a hard parse error. The
     /// server-start path converts this to `io::ErrorKind::InvalidData`.
     #[test]
-    fn parse_line_rejects_secondary_without_dash() {
+    fn parse_line_rejects_secondary_without_prefix() {
         let err = parse_line("Sellink Seelink").unwrap_err();
         match err {
-            ParseError::SecondaryWithoutDash { line, token } => {
+            ParseError::UnknownSecondaryToken { line, token } => {
                 assert_eq!(line, "Sellink Seelink");
                 assert_eq!(token, "Seelink");
             }
+            other => panic!("expected UnknownSecondaryToken, got {other:?}"),
         }
     }
 
@@ -734,7 +836,7 @@ mod tests {
     #[test]
     fn parse_line_rejects_bare_dash() {
         let err = parse_line("Sellink -").unwrap_err();
-        assert!(matches!(err, ParseError::SecondaryWithoutDash { .. }));
+        assert!(matches!(err, ParseError::UnknownSecondaryToken { .. }));
     }
 
     // --- Bypass-alias behavioural tests ---
@@ -789,6 +891,179 @@ mod tests {
         assert_eq!(g.replace("mit seelink heute."), "mit Sellink heute.");
     }
 
+    // --- Per-entry distance override (!N) parser tests ---
+
+    /// Default behaviour preserved: a single-token line yields
+    /// `max_distance = DEFAULT_MAX_EDIT_DISTANCE`.
+    #[test]
+    fn parse_line_default_distance_is_two() {
+        let entry = parse_line("Aspirin").unwrap().unwrap();
+        assert_eq!(entry.max_distance, DEFAULT_MAX_EDIT_DISTANCE);
+        assert_eq!(entry.max_distance, 2);
+    }
+
+    /// Loosen: `!3` opens an entry to DL=3 — the use case that
+    /// motivated this whole feature (Thorazemit ↔ Torasemid).
+    #[test]
+    fn parse_line_with_distance_override_three() {
+        let entry = parse_line("Torasemid !3").unwrap().unwrap();
+        assert_eq!(&*entry.canonical, "Torasemid");
+        assert_eq!(entry.max_distance, 3);
+        assert!(entry.bypass_aliases.is_empty());
+    }
+
+    /// Tighten: `!1` shrinks an entry to DL=1 — useful for short
+    /// canonicals where DL=2 admits too many neighbours.
+    #[test]
+    fn parse_line_accepts_distance_one() {
+        let entry = parse_line("Aspirin !1").unwrap().unwrap();
+        assert_eq!(entry.max_distance, 1);
+    }
+
+    /// `!N` and `-alias` are independent and may coexist on the same
+    /// line. Order does not matter to the parser.
+    #[test]
+    fn parse_line_combines_distance_and_alias() {
+        let entry = parse_line("Torasemid !3 -Thorasemid").unwrap().unwrap();
+        assert_eq!(&*entry.canonical, "Torasemid");
+        assert_eq!(entry.max_distance, 3);
+        assert!(entry.bypass_aliases.contains("thorasemid"));
+    }
+
+    /// `!N` may appear before or after `-alias` tokens — whitespace
+    /// order is irrelevant. Regression guard against accidentally
+    /// coupling the parser to token order.
+    #[test]
+    fn parse_line_alias_then_distance_works() {
+        let entry = parse_line("Torasemid -Thorasemid !3").unwrap().unwrap();
+        assert_eq!(entry.max_distance, 3);
+        assert!(entry.bypass_aliases.contains("thorasemid"));
+    }
+
+    /// Above the hard cap (`HARD_MAX_EDIT_DISTANCE = 3`) is rejected.
+    /// The cap is intentional: edit-distance 4 on typical 8-char
+    /// vocabulary entries flips half the word and is no longer a
+    /// mishearing — extending here is the wrong fix.
+    #[test]
+    fn parse_line_rejects_distance_above_cap() {
+        let err = parse_line("Aspirin !4").unwrap_err();
+        match err {
+            ParseError::DistanceOutOfRange { value, .. } => assert_eq!(value, 4),
+            other => panic!("expected DistanceOutOfRange, got {other:?}"),
+        }
+    }
+
+    /// `!0` is rejected — DL=0 is the exact-match short-circuit, so
+    /// an entry tolerating "no edits" would silently never match.
+    /// Better to fail loud than to ship a useless line.
+    #[test]
+    fn parse_line_rejects_distance_zero() {
+        let err = parse_line("Aspirin !0").unwrap_err();
+        assert!(matches!(
+            err,
+            ParseError::DistanceOutOfRange { value: 0, .. }
+        ));
+    }
+
+    /// Non-numeric `!N` fails with `InvalidDistance`, not silently.
+    #[test]
+    fn parse_line_rejects_invalid_distance_token() {
+        let err = parse_line("Aspirin !abc").unwrap_err();
+        match err {
+            ParseError::InvalidDistance { token, .. } => assert_eq!(token, "!abc"),
+            other => panic!("expected InvalidDistance, got {other:?}"),
+        }
+    }
+
+    /// More than one `!N` on the same line is ambiguous — reject
+    /// rather than guess which override "wins".
+    #[test]
+    fn parse_line_rejects_duplicate_distance() {
+        let err = parse_line("Aspirin !2 !3").unwrap_err();
+        assert!(matches!(err, ParseError::DuplicateDistanceOverride { .. }));
+    }
+
+    // --- Per-entry distance override (!N) behavioural tests ---
+
+    /// Without the override, a DL=3 token does NOT match — that is
+    /// the original conservative behaviour and must be preserved.
+    #[test]
+    fn find_candidate_dl3_no_match_without_override() {
+        let g = from_lines(&["Torasemid"]);
+        // "Thorazemit" is DL=3 from "Torasemid":
+        //   T-orasemid → Thorasemid (insert h)
+        //          s   →        z   (substitution)
+        //          d   →        t   (substitution)
+        assert_eq!(
+            g.replace("Patient nimmt Thorazemit heute."),
+            "Patient nimmt Thorazemit heute."
+        );
+    }
+
+    /// With `!3`, the same DL=3 token now resolves to the canonical.
+    /// This is the headline behaviour the feature exists for.
+    #[test]
+    fn find_candidate_dl3_match_with_per_entry_override() {
+        let g = from_lines(&["Torasemid !3"]);
+        assert_eq!(
+            g.replace("Patient nimmt Thorazemit heute."),
+            "Patient nimmt Torasemid heute."
+        );
+    }
+
+    /// DL=4 is never matched, even with `!3` — the hard cap holds.
+    /// Picking a token genuinely 4 edits away from "Torasemid":
+    /// "Thorazemett" — insert h, s→z, d→t, append t.
+    #[test]
+    fn find_candidate_dl4_never_matches_with_override_three() {
+        let g = from_lines(&["Torasemid !3"]);
+        assert_eq!(
+            g.replace("Wert von Thorazemett heute."),
+            "Wert von Thorazemett heute."
+        );
+    }
+
+    /// `!1` tightens the threshold: a DL=2 token that *would* have
+    /// matched at the default no longer does. Verifies that the
+    /// per-entry value really overrides the default in both directions.
+    #[test]
+    fn find_candidate_distance_one_tightens_filter() {
+        // Default would catch DL=2 typo of "Cerebrum"; !1 must not.
+        let g = from_lines(&["Cerebrum !1"]);
+        // "Zerabrum": z→c (1) + a→e (1) = DL=2 from "Cerebrum".
+        assert_eq!(g.replace("Blutung im Zerabrum."), "Blutung im Zerabrum.");
+        // DL=1 still matches under !1.
+        assert_eq!(g.replace("Blutung im Zerebrum."), "Blutung im Cerebrum.");
+    }
+
+    /// `!3` does NOT bypass the dict-veto. If a DL=3 candidate is
+    /// itself a real word of the language, Hunspell's veto must still
+    /// hold — the per-entry threshold widens the DL filter, nothing else.
+    #[test]
+    fn find_candidate_dl3_blocked_by_dict_veto() {
+        // Mock dict claims the DL=3 token is a real word. Even with
+        // !3 admitting it past the DL filter, the veto must keep it.
+        let g =
+            from_lines(&["Torasemid !3"]).with_dict(Box::new(HashSetDict::new(&["Thorazemit"])));
+        assert_eq!(
+            g.replace("Patient nimmt Thorazemit heute."),
+            "Patient nimmt Thorazemit heute."
+        );
+    }
+
+    /// Bypass-aliases compose with `!3`: an explicit alias still
+    /// overrides the dict-veto on a !3-entry, just as it does on a
+    /// default-distance entry.
+    #[test]
+    fn bypass_alias_works_alongside_distance_override() {
+        let g = from_lines(&["Torasemid !3 -Thorazemit"])
+            .with_dict(Box::new(HashSetDict::new(&["Thorazemit"])));
+        assert_eq!(
+            g.replace("Patient nimmt Thorazemit heute."),
+            "Patient nimmt Torasemid heute."
+        );
+    }
+
     /// `load_dir` propagates a parser error as `InvalidData`, with the
     /// offending file path in the message — server-start should fail
     /// loud rather than silently misindex.

server/vocab/vocabulary.txt

@@ -146,7 +146,7 @@ Telmisartan
 Temesta
 Thyroxin
 Timolol
-Torasemid
+Torasemid !3
 Tramadol
 Tranxilium
 Trulicity