Refactor gazetteer annotation format

The gazetteer now annotates text with `Canonical [?original]`. This
prioritizes the corrected term for the LLM while keeping the original
transcription as a fallback.

This change aligns the gazetteer's annotation strategy with the LLM's
prompt, which expects corrections to be marked for review. Previously,
the format was `original [?Canonical]`, which could lead to the LLM
using the potentially incorrect original term.
This commit is contained in:
2026-04-16 20:16:42 +02:00
parent 32f6557d85
commit bacbcb90fe
3 changed files with 30 additions and 19 deletions
+25 -14
View File
@@ -4,8 +4,10 @@
//! The LLM cannot know invented proper names (drug brands, uncommon
//! technical terms). Whisper misrenders them, usually within a couple
//! of edits of the real spelling. This module finds those tokens and
//! rewrites them as `token [?Canonical]`. The LLM decides in context
//! whether to adopt the hint.
//! rewrites them as `Canonical [?original]` — the vocabulary form
//! becomes the anchor, the original Whisper token sits in the bracket
//! as a veto option. The LLM decides in context whether to keep the
//! canonical or revert to the original.
//!
//! Matcher: single-stage linear Damerau-Levenshtein scan over all
//! entries, filtered by distance ≤ MAX_EDIT_DISTANCE. At typical sizes
@@ -91,9 +93,12 @@ impl Gazetteer {
self.entries.push(entry.into());
}
/// Annotate `text` by appending ` [?Canonical]` after any word-token
/// that's within edit-distance 2 of a gazetteer entry (and not an
/// exact match). Non-word segments pass through byte-identical.
/// Annotate `text` by replacing any word-token within edit-distance
/// 2 of a gazetteer entry with `Canonical [?original]`. The
/// canonical takes the anchor position so the LLM's default action
/// is to adopt it; the original Whisper token stays available in
/// the bracket as a veto option. Exact matches and non-word
/// segments pass through unchanged.
pub fn annotate(&self, text: &str) -> String {
if self.is_empty() {
return text.to_string();
@@ -102,10 +107,16 @@ impl Gazetteer {
for seg in tokenize(text) {
match seg {
Segment::Word(w) => match self.find_candidate(w) {
Some(c) => {
out.push_str(w);
out.push_str(" [?");
Some((c, d)) => {
tracing::info!(
token = w,
canonical = c,
distance = d,
"gazetteer hit"
);
out.push_str(c);
out.push_str(" [?");
out.push_str(w);
out.push(']');
}
None => out.push_str(w),
@@ -116,7 +127,7 @@ impl Gazetteer {
out
}
fn find_candidate(&self, token: &str) -> Option<&str> {
fn find_candidate(&self, token: &str) -> Option<(&str, usize)> {
if token.chars().count() < MIN_TOKEN_LEN {
return None;
}
@@ -130,7 +141,7 @@ impl Gazetteer {
.map(|c| (damerau_levenshtein(&lower, &c.to_lowercase()), c.as_ref()))
.filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE)
.min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(b.1)))
.map(|(_, c)| c)
.map(|(d, c)| (c, d))
}
}
@@ -191,7 +202,7 @@ mod tests {
let g = from_entries(&["Cerebrum"]);
let out = g.annotate("Patient mit Blutung im Zerebrum.");
assert!(
out.contains("Zerebrum [?Cerebrum]"),
out.contains("Cerebrum [?Zerebrum]"),
"expected annotation, got: {out:?}"
);
}
@@ -239,7 +250,7 @@ mod tests {
let g = from_entries(&["Carvedilol"]);
let out = g.annotate("Patient nimmt Carvenilol.");
assert!(
out.contains("Carvenilol [?Carvedilol]"),
out.contains("Carvedilol [?Carvenilol]"),
"got: {out:?}"
);
}
@@ -250,7 +261,7 @@ mod tests {
let g = from_entries(&["Berodual"]);
let out = g.annotate("Patient bekommt Erodual.");
assert!(
out.contains("Erodual [?Berodual]"),
out.contains("Berodual [?Erodual]"),
"got: {out:?}"
);
}
@@ -262,7 +273,7 @@ mod tests {
let g = from_entries(&["Cerebrum", "Cerebrom"]);
let out = g.annotate("Patient mit Blutung im Zerebrum.");
assert!(
out.contains("Zerebrum [?Cerebrum]"),
out.contains("Cerebrum [?Zerebrum]"),
"expected Cerebrum as closer candidate, got: {out:?}"
);
}