Remove obsolete vocabulary build scripts
The build scripts for generating `anatomy.txt` and related files have been removed as they are no longer used or maintained. The Kölner Phonetik module was also removed as it was not being utilized.
This commit is contained in:
@@ -1,238 +0,0 @@
|
||||
//! Kölner Phonetik (Postel 1969) — deterministic sound-hash for German
|
||||
//! words. Two inputs with the same code are phonetically similar.
|
||||
//!
|
||||
//! We use this to find gazetteer entries that *sound like* a
|
||||
//! misspelled token in the transcript. Example: both `Vomex` (real
|
||||
//! drug name) and `Womax` (Whisper hallucination) encode to `3648`,
|
||||
//! so a gazetteer containing `Vomex` can surface it as a candidate
|
||||
//! correction for `Womax`.
|
||||
//!
|
||||
//! Pure function, no I/O, no allocation beyond the returned `String`.
|
||||
|
||||
/// Encode `s` as a Kölner-Phonetik code. Output is a sequence of ASCII
|
||||
/// digits `0`–`8`. Non-alphabetic input chars are silently dropped;
|
||||
/// case is ignored. Empty input returns an empty string.
|
||||
///
|
||||
/// Rules (condensed from Postel 1969 / Wikipedia):
|
||||
/// - Vowels `A E I J O U Y` → `0`; leading `0` is kept, internal /
|
||||
/// trailing `0`s are removed in post-processing.
|
||||
/// - `H` is not encoded at all (neither emitted nor collapsed).
|
||||
/// - `B` → `1`; `P` → `1`, but `PH` → `3`.
|
||||
/// - `D T` → `2`, but before `C S Z` → `8`.
|
||||
/// - `F V W` → `3`.
|
||||
/// - `G K Q` → `4`.
|
||||
/// - `C` is context-dependent:
|
||||
/// * initial + before `A H K L O Q R U X` → `4`
|
||||
/// * initial otherwise → `8`
|
||||
/// * after `S Z` → `8`
|
||||
/// * before `A H K O Q U X` → `4`
|
||||
/// * otherwise → `8`
|
||||
/// - `X` → `48`, but after `C K Q` → `8` (those already encoded the 4).
|
||||
/// - `L` → `5`; `M N` → `6`; `R` → `7`; `S Z` → `8`.
|
||||
/// - Post: collapse consecutive equal digits, then drop every non-leading `0`.
|
||||
pub fn encode(s: &str) -> String {
|
||||
let prepared = prepare(s);
|
||||
if prepared.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
let raw = raw_encode(&prepared);
|
||||
postprocess(&raw)
|
||||
}
|
||||
|
||||
/// Expand umlauts + `ß`, uppercase everything else, drop non-A-Z chars.
|
||||
/// Returns a `Vec<char>` because subsequent passes need random access
|
||||
/// and look-ahead, which is awkward on `&str` (UTF-8 byte indices ≠
|
||||
/// char positions).
|
||||
fn prepare(s: &str) -> Vec<char> {
|
||||
let mut out: Vec<char> = Vec::with_capacity(s.len());
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'ä' | 'Ä' => out.push('A'),
|
||||
'ö' | 'Ö' => out.push('O'),
|
||||
'ü' | 'Ü' => out.push('U'),
|
||||
'ß' => {
|
||||
out.push('S');
|
||||
out.push('S');
|
||||
}
|
||||
_ => {
|
||||
for u in c.to_uppercase() {
|
||||
if u.is_ascii_uppercase() {
|
||||
out.push(u);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Per-character encoding with one-char look-ahead and look-behind.
|
||||
/// Emits ASCII digit chars into the output buffer. `H` emits nothing;
|
||||
/// `X` may emit two chars.
|
||||
fn raw_encode(chars: &[char]) -> Vec<char> {
|
||||
let mut out: Vec<char> = Vec::with_capacity(chars.len() + 2);
|
||||
for (i, &c) in chars.iter().enumerate() {
|
||||
let next = chars.get(i + 1).copied();
|
||||
let prev = if i > 0 { chars.get(i - 1).copied() } else { None };
|
||||
match c {
|
||||
'A' | 'E' | 'I' | 'J' | 'O' | 'U' | 'Y' => out.push('0'),
|
||||
'H' => {} // not encoded
|
||||
'B' => out.push('1'),
|
||||
'P' => out.push(if next == Some('H') { '3' } else { '1' }),
|
||||
'D' | 'T' => out.push(
|
||||
if matches!(next, Some('C') | Some('S') | Some('Z')) {
|
||||
'8'
|
||||
} else {
|
||||
'2'
|
||||
},
|
||||
),
|
||||
'F' | 'V' | 'W' => out.push('3'),
|
||||
'G' | 'K' | 'Q' => out.push('4'),
|
||||
'C' => {
|
||||
let d = if i == 0 {
|
||||
// Word-initial C.
|
||||
if matches!(
|
||||
next,
|
||||
Some('A') | Some('H') | Some('K') | Some('L')
|
||||
| Some('O') | Some('Q') | Some('R')
|
||||
| Some('U') | Some('X')
|
||||
) {
|
||||
'4'
|
||||
} else {
|
||||
'8'
|
||||
}
|
||||
} else if matches!(prev, Some('S') | Some('Z')) {
|
||||
// Override: after S/Z always 8, regardless of next.
|
||||
'8'
|
||||
} else if matches!(
|
||||
next,
|
||||
Some('A') | Some('H') | Some('K') | Some('O')
|
||||
| Some('Q') | Some('U') | Some('X')
|
||||
) {
|
||||
'4'
|
||||
} else {
|
||||
'8'
|
||||
};
|
||||
out.push(d);
|
||||
}
|
||||
'X' => {
|
||||
if matches!(prev, Some('C') | Some('K') | Some('Q')) {
|
||||
// Previous char already emitted the '4'; only the '8' is new.
|
||||
out.push('8');
|
||||
} else {
|
||||
out.push('4');
|
||||
out.push('8');
|
||||
}
|
||||
}
|
||||
'L' => out.push('5'),
|
||||
'M' | 'N' => out.push('6'),
|
||||
'R' => out.push('7'),
|
||||
'S' | 'Z' => out.push('8'),
|
||||
_ => {} // unreachable after prepare(), but harmless.
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Collapse runs of identical digits into one, then drop every internal
|
||||
/// or trailing `0`. A leading `0` is preserved because it carries
|
||||
/// information — the word started with a vowel.
|
||||
fn postprocess(raw: &[char]) -> String {
|
||||
let mut dedup: Vec<char> = Vec::with_capacity(raw.len());
|
||||
for &c in raw {
|
||||
if dedup.last() != Some(&c) {
|
||||
dedup.push(c);
|
||||
}
|
||||
}
|
||||
let mut out = String::with_capacity(dedup.len());
|
||||
for (i, &c) in dedup.iter().enumerate() {
|
||||
if c == '0' && i > 0 {
|
||||
continue;
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn encodes_classic_german_surnames() {
|
||||
// Reference values from the German Wikipedia article on Kölner Phonetik.
|
||||
assert_eq!(encode("Müller"), "657");
|
||||
assert_eq!(encode("Meier"), "67");
|
||||
assert_eq!(encode("Meyer"), "67");
|
||||
assert_eq!(encode("Mayer"), "67");
|
||||
assert_eq!(encode("Schmidt"), "862");
|
||||
assert_eq!(encode("Schmitz"), "868");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn treats_umlaut_and_digraph_as_equivalent() {
|
||||
// Classic test: "ü" and "ue" should normalise to the same code.
|
||||
assert_eq!(encode("Müller"), encode("Mueller"));
|
||||
}
|
||||
|
||||
/// Regression guard for the ASR-correction use case. Womax (Whisper
|
||||
/// hallucination) and Vomex (real drug name) must encode identically
|
||||
/// — that is the mechanism that makes the gazetteer lookup fire.
|
||||
#[test]
|
||||
fn vomex_and_womax_collide() {
|
||||
assert_eq!(encode("Vomex"), encode("Womax"));
|
||||
assert_eq!(encode("Vomex"), "3648");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_and_non_alphabetic_input_yields_empty_code() {
|
||||
assert_eq!(encode(""), "");
|
||||
assert_eq!(encode("123 !?"), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_case_insensitive() {
|
||||
assert_eq!(encode("MÜLLER"), encode("müller"));
|
||||
assert_eq!(encode("MüLLeR"), encode("Müller"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leading_vowel_keeps_leading_zero() {
|
||||
// "Aber" → A=0, B=1, E=0, R=7 → "0107" → dedupe: "0107" → drop non-leading 0: "017".
|
||||
assert_eq!(encode("Aber"), "017");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn c_before_h_at_word_start_codes_4() {
|
||||
// "Christian": C(initial,next=H→4) H(skip) R=7 I=0 S=8 T(next=I,not CSZ→2) I=0 A=0 N=6
|
||||
// raw = 4,7,0,8,2,0,0,6 → dedupe = 4,7,0,8,2,0,6 → drop internal 0 = 4,7,8,2,6
|
||||
assert_eq!(encode("Christian"), "47826");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x_after_c_is_single_digit() {
|
||||
// "Axel": A=0, X (prev=A, emits 4+8), E=0, L=5 → 0,4,8,0,5 → dedupe same → "0485"
|
||||
assert_eq!(encode("Axel"), "0485");
|
||||
// "Knixel": K=4, N=6, I=0, X (prev=I, emits 4+8), E=0, L=5 → 4,6,0,4,8,0,5 → "46485"
|
||||
assert_eq!(encode("Knixel"), "46485");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_embedded_digits_and_punctuation() {
|
||||
// "F1-Chip" → prepared: F,C,H,I,P. F=3, C(non-initial,prev=F,next=H→4),
|
||||
// H(skip), I=0, P(no next=H→1). Raw: 3,4,0,1 → no dedupe → "341".
|
||||
assert_eq!(encode("F1-Chip"), "341");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ph_at_start_codes_3() {
|
||||
// "Phil" → P(next=H→3), H(skip), I=0, L=5 → 3,0,5 → "35".
|
||||
assert_eq!(encode("Phil"), "35");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s_z_before_c_yields_8() {
|
||||
// "Katze" → K=4, A=0, T(next=Z→8), Z=8, E=0 → 4,0,8,8,0 → dedupe 4,0,8,0 → "48".
|
||||
assert_eq!(encode("Katze"), "48");
|
||||
}
|
||||
}
|
||||
+54
-110
@@ -1,32 +1,33 @@
|
||||
//! Gazetteer: domain-vocabulary lookup that injects correction hints
|
||||
//! before the analysis LLM runs. See
|
||||
//! `~/.claude/plans/happy-mapping-snail.md` for the design document.
|
||||
//! before the analysis LLM runs.
|
||||
//!
|
||||
//! The LLM cannot know invented proper names (drug brands, anatomical
|
||||
//! Latin/Greek terms). Whisper misrenders them phonetically
|
||||
//! (`Vomex` → `Womax`). This module exposes a static lookup that, given
|
||||
//! a transcript, finds tokens that are likely typos of a known entry
|
||||
//! and rewrites them as `token [?Canonical]`. The LLM decides in context
|
||||
//! The LLM cannot know invented proper names (drug brands, uncommon
|
||||
//! technical terms). Whisper misrenders them, usually within a couple
|
||||
//! of edits of the real spelling. This module finds those tokens and
|
||||
//! rewrites them as `token [?Canonical]`. The LLM decides in context
|
||||
//! whether to adopt the hint.
|
||||
//!
|
||||
//! Matcher: single-stage linear Damerau-Levenshtein scan over all
|
||||
//! entries, filtered by distance ≤ MAX_EDIT_DISTANCE. At typical sizes
|
||||
//! (~200 entries × ~50 transcript tokens), the per-pass cost stays well
|
||||
//! under a millisecond — dwarfed by the downstream LLM call.
|
||||
|
||||
pub mod koelner;
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashSet;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
use strsim::damerau_levenshtein;
|
||||
|
||||
/// Minimum char count for *both* gazetteer entries and transcript tokens.
|
||||
/// Calibrated against curated medication lists: 5 keeps prototypical
|
||||
/// 5-char brand names (Vomex, Tavor, Lasix, Nasic) while still blocking
|
||||
/// the 4-char deutsche Alltagswort ↔ eponym collisions that unkurated
|
||||
/// crawls produce (Bein↔Behn, nach↔Nuck).
|
||||
/// Calibrated against curated vocabulary: 5 keeps prototypical 5-char
|
||||
/// brand names (Vomex, Tavor, Lasix) while still blocking 4-char
|
||||
/// collisions between everyday German words and short proper nouns
|
||||
/// (Bein↔Behn, nach↔Nuck).
|
||||
const MIN_TOKEN_LEN: usize = 5;
|
||||
|
||||
/// Maximum Damerau-Levenshtein distance for a candidate to qualify as
|
||||
/// a typo of a gazetteer entry. Distance 0 is excluded — the `exact`
|
||||
/// set short-circuits that case.
|
||||
/// Maximum Damerau-Levenshtein distance for a candidate to qualify as a
|
||||
/// typo of a gazetteer entry. Distance 0 is handled by the `exact`
|
||||
/// short-circuit; distances > 2 are rejected as too speculative.
|
||||
const MAX_EDIT_DISTANCE: usize = 2;
|
||||
|
||||
/// Domain-specific vocabulary, populated once at server start from
|
||||
@@ -34,23 +35,22 @@ const MAX_EDIT_DISTANCE: usize = 2;
|
||||
/// via `Arc<Gazetteer>`.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Gazetteer {
|
||||
/// Kölner-Phonetik code → canonical entries sharing that code. The
|
||||
/// canonicals keep their original casing for display.
|
||||
by_phonetic: HashMap<String, Vec<Box<str>>>,
|
||||
/// Canonical entries in original casing — source of truth for both
|
||||
/// scanning and emission.
|
||||
entries: Vec<Box<str>>,
|
||||
/// Lowercased canonicals; short-circuits annotation when the token
|
||||
/// is already spelled correctly.
|
||||
/// already matches a known entry exactly.
|
||||
exact: HashSet<Box<str>>,
|
||||
}
|
||||
|
||||
impl Gazetteer {
|
||||
/// Empty gazetteer. `annotate` is a pure pass-through on this instance.
|
||||
pub fn empty() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Load every `*.txt` file (non-recursive) under `dir`. Blank lines
|
||||
/// and lines beginning with `#` are skipped. Case-insensitive
|
||||
/// duplicates are silently collapsed.
|
||||
/// duplicates across files are silently collapsed.
|
||||
pub async fn load_dir(dir: &Path) -> io::Result<Self> {
|
||||
let mut g = Self::default();
|
||||
let mut entries = tokio::fs::read_dir(dir).await?;
|
||||
@@ -71,40 +71,29 @@ impl Gazetteer {
|
||||
Ok(g)
|
||||
}
|
||||
|
||||
/// Number of unique entries (case-insensitive).
|
||||
pub fn len(&self) -> usize {
|
||||
self.exact.len()
|
||||
self.entries.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.exact.is_empty()
|
||||
self.entries.is_empty()
|
||||
}
|
||||
|
||||
fn insert(&mut self, entry: &str) {
|
||||
// Skip short entries — they cause noisy phonetic collisions with
|
||||
// ordinary German words. Same threshold as token matching.
|
||||
if entry.chars().count() < MIN_TOKEN_LEN {
|
||||
return;
|
||||
}
|
||||
let lower = entry.to_lowercase();
|
||||
if !self.exact.insert(lower.into_boxed_str()) {
|
||||
// Duplicate (case-insensitive) — don't add to phonetic index either.
|
||||
// Duplicate (case-insensitive) — already indexed.
|
||||
return;
|
||||
}
|
||||
let code = koelner::encode(entry);
|
||||
if code.is_empty() {
|
||||
return;
|
||||
}
|
||||
self.by_phonetic
|
||||
.entry(code)
|
||||
.or_default()
|
||||
.push(entry.into());
|
||||
self.entries.push(entry.into());
|
||||
}
|
||||
|
||||
/// Annotate `text` by appending ` [?Canonical]` after any word-token
|
||||
/// that looks like a misspelling of a gazetteer entry. Non-matching
|
||||
/// tokens and all non-word segments (whitespace, punctuation, digits)
|
||||
/// pass through byte-identical.
|
||||
/// that's within edit-distance 2 of a gazetteer entry (and not an
|
||||
/// exact match). Non-word segments pass through byte-identical.
|
||||
pub fn annotate(&self, text: &str) -> String {
|
||||
if self.is_empty() {
|
||||
return text.to_string();
|
||||
@@ -136,45 +125,15 @@ impl Gazetteer {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Stage 1: phonetic-indexed lookup. Fast, filters out the vast
|
||||
// majority of non-matches in O(1). Catches same-code neighbours
|
||||
// like Klexane↔Clexane, Diacepam↔Diazepam.
|
||||
let code = koelner::encode(token);
|
||||
if !code.is_empty()
|
||||
&& let Some(candidates) = self.by_phonetic.get(&code)
|
||||
&& let Some(hit) = best_candidate(&lower, candidates.iter().map(|c| c.as_ref()))
|
||||
{
|
||||
return Some(hit);
|
||||
}
|
||||
|
||||
// Stage 2: brute-force edit-distance over every entry. Catches
|
||||
// ASR errors where Whisper substitutes phonetically-distinct
|
||||
// consonants (Carvenilol↔Carvedilol: N↔D) or drops a letter
|
||||
// that shifts the Kölner code (Acoxia↔Arcoxia: missing R).
|
||||
// O(n) per unknown token; at n≈200 entries and ~50 tokens per
|
||||
// transcript the absolute cost stays well under a millisecond.
|
||||
best_candidate(
|
||||
&lower,
|
||||
self.by_phonetic.values().flatten().map(|c| c.as_ref()),
|
||||
)
|
||||
self.entries
|
||||
.iter()
|
||||
.map(|c| (damerau_levenshtein(&lower, &c.to_lowercase()), c.as_ref()))
|
||||
.filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE)
|
||||
.min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(b.1)))
|
||||
.map(|(_, c)| c)
|
||||
}
|
||||
}
|
||||
|
||||
/// Select the canonical entry from `candidates` that is closest to
|
||||
/// `lower_token` by Damerau-Levenshtein distance, within the allowed
|
||||
/// threshold. Distance 0 is rejected because that's already handled by
|
||||
/// the `exact` short-circuit. Ties break alphabetically for determinism.
|
||||
fn best_candidate<'a>(
|
||||
lower_token: &str,
|
||||
candidates: impl Iterator<Item = &'a str>,
|
||||
) -> Option<&'a str> {
|
||||
candidates
|
||||
.map(|c| (damerau_levenshtein(lower_token, &c.to_lowercase()), c))
|
||||
.filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE)
|
||||
.min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(b.1)))
|
||||
.map(|(_, c)| c)
|
||||
}
|
||||
|
||||
/// A slice of the input text — either a word or a non-word run.
|
||||
enum Segment<'a> {
|
||||
Word(&'a str),
|
||||
@@ -212,8 +171,6 @@ mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
/// Build a gazetteer from an in-memory list of canonical entries —
|
||||
/// avoids filesystem I/O in the matcher tests.
|
||||
fn from_entries(entries: &[&str]) -> Gazetteer {
|
||||
let mut g = Gazetteer::empty();
|
||||
for e in entries {
|
||||
@@ -229,12 +186,8 @@ mod tests {
|
||||
assert_eq!(g.annotate(input), input);
|
||||
}
|
||||
|
||||
/// Core regression: a token that's a phonetic neighbor at edit
|
||||
/// distance ≤ 2 gets a hint appended. Cerebrum and Zerebrum both
|
||||
/// encode to Kölner `87176` and differ by one char — a plausible
|
||||
/// Whisper-style transcription slip.
|
||||
#[test]
|
||||
fn annotates_typo_for_phonetic_neighbor() {
|
||||
fn annotates_typo_within_distance() {
|
||||
let g = from_entries(&["Cerebrum"]);
|
||||
let out = g.annotate("Patient mit Blutung im Zerebrum.");
|
||||
assert!(
|
||||
@@ -248,18 +201,18 @@ mod tests {
|
||||
let g = from_entries(&["Cerebrum"]);
|
||||
assert_eq!(
|
||||
g.annotate("Patient mit Blutung im Cerebrum."),
|
||||
"Patient mit Blutung im Cerebrum.",
|
||||
"Patient mit Blutung im Cerebrum."
|
||||
);
|
||||
}
|
||||
|
||||
/// Lowercase variant of an entry is still "exact" — we never nag
|
||||
/// the LLM with a hint for a token that's only case-different.
|
||||
/// Lowercase variant of an entry is still "exact" — we never nag the
|
||||
/// LLM with a hint for a token that differs only in case.
|
||||
#[test]
|
||||
fn noop_on_case_only_difference() {
|
||||
let g = from_entries(&["Cerebrum"]);
|
||||
assert_eq!(
|
||||
g.annotate("Patient mit Blutung im cerebrum."),
|
||||
"Patient mit Blutung im cerebrum.",
|
||||
"Patient mit Blutung im cerebrum."
|
||||
);
|
||||
}
|
||||
|
||||
@@ -270,51 +223,42 @@ mod tests {
|
||||
assert_eq!(g.annotate(input), input);
|
||||
}
|
||||
|
||||
/// Gazetteer entries shorter than MIN_TOKEN_LEN are rejected at load
|
||||
/// time. This prevents 4-char eponyms (Nuck, Behn, Bell, …) from
|
||||
/// generating noisy false-positive matches against everyday German
|
||||
/// words — the empirical reason MIN_TOKEN_LEN exists.
|
||||
/// Entries shorter than MIN_TOKEN_LEN are rejected at load time —
|
||||
/// prevents short-proper-noun collisions with everyday German.
|
||||
#[test]
|
||||
fn rejects_gazetteer_entries_below_min_length() {
|
||||
fn rejects_entries_below_min_length() {
|
||||
let g = from_entries(&["Nuck", "Bein", "Behn"]);
|
||||
assert_eq!(g.len(), 0, "short entries must be dropped");
|
||||
// Input with a matching short word passes through untouched.
|
||||
let input = "Patient klagt nach schwerem Heben.";
|
||||
assert_eq!(g.annotate(input), input);
|
||||
}
|
||||
|
||||
/// Edit-distance fallback: Whisper substitutes phonetically-distinct
|
||||
/// consonants (here N↔D in Carvenilol↔Carvedilol). The Kölner codes
|
||||
/// differ (N=6 vs D=2), so stage-1 misses — the brute-force stage
|
||||
/// must catch the distance-1 match.
|
||||
/// Classic ASR-substitution case (N↔D) — distance 1 match.
|
||||
#[test]
|
||||
fn edit_distance_fallback_catches_consonant_substitution() {
|
||||
fn annotates_consonant_substitution() {
|
||||
let g = from_entries(&["Carvedilol"]);
|
||||
let out = g.annotate("Patient nimmt Carvenilol.");
|
||||
assert!(
|
||||
out.contains("Carvenilol [?Carvedilol]"),
|
||||
"expected fallback annotation, got: {out:?}"
|
||||
"got: {out:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Edit-distance fallback: Whisper drops a consonant (leading B in
|
||||
/// Berodual). Kölner codes are completely different — only the
|
||||
/// linear edit-distance scan saves the match.
|
||||
/// Dropped leading consonant — edit-distance 1 from Berodual.
|
||||
#[test]
|
||||
fn edit_distance_fallback_catches_dropped_consonant() {
|
||||
fn annotates_dropped_consonant() {
|
||||
let g = from_entries(&["Berodual"]);
|
||||
let out = g.annotate("Patient bekommt Erodual.");
|
||||
assert!(
|
||||
out.contains("Erodual [?Berodual]"),
|
||||
"expected fallback annotation, got: {out:?}"
|
||||
"got: {out:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// With two phonetically-colliding candidates, the one with smaller
|
||||
/// Damerau-Levenshtein distance wins. Zerebrum↔Cerebrum = 1,
|
||||
/// Zerebrum↔Cerebrom = 2 → Cerebrum is chosen.
|
||||
/// With multiple in-range candidates, the closest by
|
||||
/// Damerau-Levenshtein wins; ties break alphabetically.
|
||||
#[test]
|
||||
fn picks_closest_candidate_by_edit_distance() {
|
||||
fn picks_closest_candidate() {
|
||||
let g = from_entries(&["Cerebrum", "Cerebrom"]);
|
||||
let out = g.annotate("Patient mit Blutung im Zerebrum.");
|
||||
assert!(
|
||||
@@ -336,7 +280,7 @@ mod tests {
|
||||
tokio::fs::write(dir.path().join("meds.txt"), "Aspirin\nIbuprofen\n")
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::fs::write(dir.path().join("anatomy.txt"), "Coracoideus\n")
|
||||
tokio::fs::write(dir.path().join("more.txt"), "Coracoideus\n")
|
||||
.await
|
||||
.unwrap();
|
||||
let g = Gazetteer::load_dir(dir.path()).await.unwrap();
|
||||
@@ -348,7 +292,7 @@ mod tests {
|
||||
let dir = tempdir().unwrap();
|
||||
tokio::fs::write(
|
||||
dir.path().join("v.txt"),
|
||||
"# medications\nAspirin\n\n \n# another\nIbuprofen\n",
|
||||
"# meds\nAspirin\n\n \n# another\nIbuprofen\n",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
Reference in New Issue
Block a user