Refactor gazetteer annotation format
The gazetteer now annotates text with `Canonical [?original]`. This prioritizes the corrected term for the LLM while keeping the original transcription as a fallback. This change aligns the gazetteer's annotation strategy with the LLM's prompt, which expects corrections to be marked for review. Previously, the format was `original [?Canonical]`, which could lead to the LLM using the potentially incorrect original term.
This commit is contained in:
+25
-14
@@ -4,8 +4,10 @@
|
||||
//! The LLM cannot know invented proper names (drug brands, uncommon
|
||||
//! technical terms). Whisper misrenders them, usually within a couple
|
||||
//! of edits of the real spelling. This module finds those tokens and
|
||||
//! rewrites them as `token [?Canonical]`. The LLM decides in context
|
||||
//! whether to adopt the hint.
|
||||
//! rewrites them as `Canonical [?original]` — the vocabulary form
|
||||
//! becomes the anchor, the original Whisper token sits in the bracket
|
||||
//! as a veto option. The LLM decides in context whether to keep the
|
||||
//! canonical or revert to the original.
|
||||
//!
|
||||
//! Matcher: single-stage linear Damerau-Levenshtein scan over all
|
||||
//! entries, filtered by distance ≤ MAX_EDIT_DISTANCE. At typical sizes
|
||||
@@ -91,9 +93,12 @@ impl Gazetteer {
|
||||
self.entries.push(entry.into());
|
||||
}
|
||||
|
||||
/// Annotate `text` by appending ` [?Canonical]` after any word-token
|
||||
/// that's within edit-distance 2 of a gazetteer entry (and not an
|
||||
/// exact match). Non-word segments pass through byte-identical.
|
||||
/// Annotate `text` by replacing any word-token within edit-distance
|
||||
/// 2 of a gazetteer entry with `Canonical [?original]`. The
|
||||
/// canonical takes the anchor position so the LLM's default action
|
||||
/// is to adopt it; the original Whisper token stays available in
|
||||
/// the bracket as a veto option. Exact matches and non-word
|
||||
/// segments pass through unchanged.
|
||||
pub fn annotate(&self, text: &str) -> String {
|
||||
if self.is_empty() {
|
||||
return text.to_string();
|
||||
@@ -102,10 +107,16 @@ impl Gazetteer {
|
||||
for seg in tokenize(text) {
|
||||
match seg {
|
||||
Segment::Word(w) => match self.find_candidate(w) {
|
||||
Some(c) => {
|
||||
out.push_str(w);
|
||||
out.push_str(" [?");
|
||||
Some((c, d)) => {
|
||||
tracing::info!(
|
||||
token = w,
|
||||
canonical = c,
|
||||
distance = d,
|
||||
"gazetteer hit"
|
||||
);
|
||||
out.push_str(c);
|
||||
out.push_str(" [?");
|
||||
out.push_str(w);
|
||||
out.push(']');
|
||||
}
|
||||
None => out.push_str(w),
|
||||
@@ -116,7 +127,7 @@ impl Gazetteer {
|
||||
out
|
||||
}
|
||||
|
||||
fn find_candidate(&self, token: &str) -> Option<&str> {
|
||||
fn find_candidate(&self, token: &str) -> Option<(&str, usize)> {
|
||||
if token.chars().count() < MIN_TOKEN_LEN {
|
||||
return None;
|
||||
}
|
||||
@@ -130,7 +141,7 @@ impl Gazetteer {
|
||||
.map(|c| (damerau_levenshtein(&lower, &c.to_lowercase()), c.as_ref()))
|
||||
.filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE)
|
||||
.min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(b.1)))
|
||||
.map(|(_, c)| c)
|
||||
.map(|(d, c)| (c, d))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,7 +202,7 @@ mod tests {
|
||||
let g = from_entries(&["Cerebrum"]);
|
||||
let out = g.annotate("Patient mit Blutung im Zerebrum.");
|
||||
assert!(
|
||||
out.contains("Zerebrum [?Cerebrum]"),
|
||||
out.contains("Cerebrum [?Zerebrum]"),
|
||||
"expected annotation, got: {out:?}"
|
||||
);
|
||||
}
|
||||
@@ -239,7 +250,7 @@ mod tests {
|
||||
let g = from_entries(&["Carvedilol"]);
|
||||
let out = g.annotate("Patient nimmt Carvenilol.");
|
||||
assert!(
|
||||
out.contains("Carvenilol [?Carvedilol]"),
|
||||
out.contains("Carvedilol [?Carvenilol]"),
|
||||
"got: {out:?}"
|
||||
);
|
||||
}
|
||||
@@ -250,7 +261,7 @@ mod tests {
|
||||
let g = from_entries(&["Berodual"]);
|
||||
let out = g.annotate("Patient bekommt Erodual.");
|
||||
assert!(
|
||||
out.contains("Erodual [?Berodual]"),
|
||||
out.contains("Berodual [?Erodual]"),
|
||||
"got: {out:?}"
|
||||
);
|
||||
}
|
||||
@@ -262,7 +273,7 @@ mod tests {
|
||||
let g = from_entries(&["Cerebrum", "Cerebrom"]);
|
||||
let out = g.annotate("Patient mit Blutung im Zerebrum.");
|
||||
assert!(
|
||||
out.contains("Zerebrum [?Cerebrum]"),
|
||||
out.contains("Cerebrum [?Zerebrum]"),
|
||||
"expected Cerebrum as closer candidate, got: {out:?}"
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user