Refactor gazetteer to replace post-AI
The gazetteer has been refactored to act as a post-AI normalization filter. Previously, it was used to annotate LLM input with potential corrections from a vocabulary. This approach was ineffective because LLMs often override such hints. The new approach applies the gazetteer *after* the LLM has generated its output. This allows for deterministic correction of known terminology, including fixing LLM output drift (e.g., anglicized drug names). Key changes: - `annotate` function renamed to `replace`. - The output format changes from `Canonical [?original]` to simply the `Canonical` form. - The gazetteer now operates on the final LLM output before persistence, ensuring consistency. - Tests have been updated to reflect this new behavior, focusing on the final output rather than the LLM request payload.
This commit is contained in:
@@ -2,7 +2,7 @@ use super::AnalysisInput;
|
||||
|
||||
/// System prompt for the consolidation LLM. From docs/projektplan.md:365-369.
|
||||
/// Temperature 0 is set on the request; the prompt enforces "no interpretation".
|
||||
pub const SYSTEM_PROMPT: &str = "Du bist ein medizinischer Assistent. Fasse die folgenden diktierten Abschnitte zu einem zusammenhängenden Text zusammen. Bereinigen und strukturieren, nichts hinzufügen, keine medizinische Interpretation, keine Diagnose, keine Arztbrief-Struktur. Entferne Redundanzen. Spätere Aufnahmen haben Vorrang — Korrekturen, Nachträge und Widersprüche zugunsten der chronologisch letzten Aussage auflösen. Antworte auf Deutsch. Nur der zusammengefasste Text, keine Einleitung, kein Abschlusssatz.\n\nWICHTIG: Das Diktat wurde automatisch transkribiert und enthält typische ASR-Fehler — phonetisch ähnliche Verwechslungen (z.B. \"Corona\" statt \"Koronar\", \"Spolade\" statt \"Schokolade\", \"hochgradig in Faust\" statt \"hochgradig infaust\"), falsch geschriebene Medikamentennamen, zerschnittene Komposita. Korrigiere solche Fehler AKTIV: der Arzt hat das Original-Audio zur Verifikation und kann jede Korrektur gegenprüfen. Bei sicherer Korrektur: schreibe das korrigierte Wort. Bei Unsicherheit, wie die Korrektur lauten müsste: behalte das Original bei.\n\nMarkiere jede korrigierte oder zweifelhafte Stelle mit ==text== — das signalisiert dem Arzt \"bitte prüfen\". Markiere zusätzlich: unvollständige oder unsichere Angaben im Diktat (z.B. \"Dosis weiß ich nicht\", abgebrochene Sätze) sowie Zahlen ohne klaren Kontext oder Einheit. Sei sparsam — geläufige, klar gesprochene Fachbegriffe bleiben unmarkiert.\n\nDer Text kann Annotationen der Form `Kandidat [?Original]` enthalten — das sind Korrekturen aus einem Fach-Wörterbuch. `Kandidat` ist die vorgeschlagene Korrektur, `Original` das, was Whisper ursprünglich transkribiert hat. Prüfe mit dem Satzkontext: passt der Kandidat, übernimm ihn und markiere mit ==. Passt er nicht, verwende stattdessen das Original ohne Markierung. Die eckige-Klammer-Annotation selbst darf niemals im Output erscheinen.";
|
||||
pub const SYSTEM_PROMPT: &str = "Du bist ein medizinischer Assistent. Fasse die folgenden diktierten Abschnitte zu einem zusammenhängenden Text zusammen. Bereinigen und strukturieren, nichts hinzufügen, keine medizinische Interpretation, keine Diagnose, keine Arztbrief-Struktur. Entferne Redundanzen. Spätere Aufnahmen haben Vorrang — Korrekturen, Nachträge und Widersprüche zugunsten der chronologisch letzten Aussage auflösen. Antworte auf Deutsch. Nur der zusammengefasste Text, keine Einleitung, kein Abschlusssatz.\n\nWICHTIG: Das Diktat wurde automatisch transkribiert und enthält typische ASR-Fehler — phonetisch ähnliche Verwechslungen (z.B. \"Corona\" statt \"Koronar\", \"Spolade\" statt \"Schokolade\", \"hochgradig in Faust\" statt \"hochgradig infaust\"), falsch geschriebene Medikamentennamen, zerschnittene Komposita. Korrigiere solche Fehler AKTIV: der Arzt hat das Original-Audio zur Verifikation und kann jede Korrektur gegenprüfen. Bei sicherer Korrektur: schreibe das korrigierte Wort. Bei Unsicherheit, wie die Korrektur lauten müsste: behalte das Original bei.\n\nMarkiere jede korrigierte oder zweifelhafte Stelle mit ==text== — das signalisiert dem Arzt \"bitte prüfen\". Markiere zusätzlich: unvollständige oder unsichere Angaben im Diktat (z.B. \"Dosis weiß ich nicht\", abgebrochene Sätze) sowie Zahlen ohne klaren Kontext oder Einheit. Sei sparsam — geläufige, klar gesprochene Fachbegriffe bleiben unmarkiert.";
|
||||
|
||||
/// Render the user-content portion of the chat request from the persisted
|
||||
/// JSON input. Recordings with blank text are skipped — they carry no signal
|
||||
|
||||
@@ -78,12 +78,7 @@ async fn process(
|
||||
return;
|
||||
}
|
||||
|
||||
// Render the raw transcript bundle, then inject gazetteer hints for
|
||||
// tokens that phonetically resemble known proper names. The LLM sees
|
||||
// annotations of the form `Token [?Candidate]` and decides in context
|
||||
// whether to adopt them.
|
||||
let user_content = prompt::render_prompt(&input);
|
||||
let user_content = vocab.annotate(&user_content);
|
||||
let total_chars = user_content.chars().count();
|
||||
info!(
|
||||
case = %case_dir.display(),
|
||||
@@ -120,6 +115,11 @@ async fn process(
|
||||
}
|
||||
};
|
||||
|
||||
// Post-AI terminology normalization: enforce vocab-canonical forms
|
||||
// over the LLM's training priors (e.g. rewrite anglicized INNs like
|
||||
// `Amiodarone` back to `Amiodaron`).
|
||||
let document = vocab.replace(&document);
|
||||
|
||||
if let Err(e) = write_atomic(&document_path, document.as_bytes()).await {
|
||||
error!(path = %document_path.display(), error = %e, "writing document failed");
|
||||
return;
|
||||
|
||||
+57
-50
@@ -1,18 +1,24 @@
|
||||
//! Gazetteer: domain-vocabulary lookup that injects correction hints
|
||||
//! before the analysis LLM runs.
|
||||
//! Gazetteer: deterministic terminology normalization filter applied
|
||||
//! to every AI output (Whisper, oneliner LLM, analysis LLM) before
|
||||
//! persistence.
|
||||
//!
|
||||
//! The LLM cannot know invented proper names (drug brands, uncommon
|
||||
//! technical terms). Whisper misrenders them, usually within a couple
|
||||
//! of edits of the real spelling. This module finds those tokens and
|
||||
//! rewrites them as `Canonical [?original]` — the vocabulary form
|
||||
//! becomes the anchor, the original Whisper token sits in the bracket
|
||||
//! as a veto option. The LLM decides in context whether to keep the
|
||||
//! canonical or revert to the original.
|
||||
//! Invariant: every AI-produced text passes through `replace()` before
|
||||
//! being written to disk or forwarded downstream. Tokens within
|
||||
//! edit-distance 2 of a curated vocabulary entry are silently rewritten
|
||||
//! to the canonical form. No hints, no annotations, no LLM veto — the
|
||||
//! vocab is authoritative.
|
||||
//!
|
||||
//! Rationale: LLMs have strong training priors that override contextual
|
||||
//! hints — e.g. they anglicize `Amiodaron` to `Amiodarone` even when
|
||||
//! the input was correct. A pre-LLM hint channel cannot fix this
|
||||
//! because the drift happens in the LLM's output. Running the filter
|
||||
//! post-AI catches both Whisper typos and LLM drift with a single
|
||||
//! mechanism.
|
||||
//!
|
||||
//! Matcher: single-stage linear Damerau-Levenshtein scan over all
|
||||
//! entries, filtered by distance ≤ MAX_EDIT_DISTANCE. At typical sizes
|
||||
//! (~200 entries × ~50 transcript tokens), the per-pass cost stays well
|
||||
//! under a millisecond — dwarfed by the downstream LLM call.
|
||||
//! (~200 entries × ~50 tokens), the per-pass cost stays well under a
|
||||
//! millisecond — negligible next to the AI call it follows.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::io;
|
||||
@@ -40,7 +46,7 @@ pub struct Gazetteer {
|
||||
/// Canonical entries in original casing — source of truth for both
|
||||
/// scanning and emission.
|
||||
entries: Vec<Box<str>>,
|
||||
/// Lowercased canonicals; short-circuits annotation when the token
|
||||
/// Lowercased canonicals; short-circuits replacement when the token
|
||||
/// already matches a known entry exactly.
|
||||
exact: HashSet<Box<str>>,
|
||||
}
|
||||
@@ -93,13 +99,12 @@ impl Gazetteer {
|
||||
self.entries.push(entry.into());
|
||||
}
|
||||
|
||||
/// Annotate `text` by replacing any word-token within edit-distance
|
||||
/// 2 of a gazetteer entry with `Canonical [?original]`. The
|
||||
/// canonical takes the anchor position so the LLM's default action
|
||||
/// is to adopt it; the original Whisper token stays available in
|
||||
/// the bracket as a veto option. Exact matches and non-word
|
||||
/// segments pass through unchanged.
|
||||
pub fn annotate(&self, text: &str) -> String {
|
||||
/// Rewrite every word-token within edit-distance 2 of a gazetteer
|
||||
/// entry to the canonical form (original casing from the vocab
|
||||
/// file). Exact matches and non-word segments pass through
|
||||
/// unchanged. Each replacement is logged at info level as an
|
||||
/// audit trail.
|
||||
pub fn replace(&self, text: &str) -> String {
|
||||
if self.is_empty() {
|
||||
return text.to_string();
|
||||
}
|
||||
@@ -112,12 +117,9 @@ impl Gazetteer {
|
||||
token = w,
|
||||
canonical = c,
|
||||
distance = d,
|
||||
"gazetteer hit"
|
||||
"gazetteer replaced"
|
||||
);
|
||||
out.push_str(c);
|
||||
out.push_str(" [?");
|
||||
out.push_str(w);
|
||||
out.push(']');
|
||||
}
|
||||
None => out.push_str(w),
|
||||
},
|
||||
@@ -194,16 +196,15 @@ mod tests {
|
||||
fn empty_gazetteer_is_pass_through() {
|
||||
let g = Gazetteer::empty();
|
||||
let input = "Patient nahm Zerebrum-Medikation.";
|
||||
assert_eq!(g.annotate(input), input);
|
||||
assert_eq!(g.replace(input), input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn annotates_typo_within_distance() {
|
||||
fn replaces_typo_within_distance() {
|
||||
let g = from_entries(&["Cerebrum"]);
|
||||
let out = g.annotate("Patient mit Blutung im Zerebrum.");
|
||||
assert!(
|
||||
out.contains("Cerebrum [?Zerebrum]"),
|
||||
"expected annotation, got: {out:?}"
|
||||
assert_eq!(
|
||||
g.replace("Patient mit Blutung im Zerebrum."),
|
||||
"Patient mit Blutung im Cerebrum."
|
||||
);
|
||||
}
|
||||
|
||||
@@ -211,27 +212,36 @@ mod tests {
|
||||
fn noop_on_exact_match() {
|
||||
let g = from_entries(&["Cerebrum"]);
|
||||
assert_eq!(
|
||||
g.annotate("Patient mit Blutung im Cerebrum."),
|
||||
g.replace("Patient mit Blutung im Cerebrum."),
|
||||
"Patient mit Blutung im Cerebrum."
|
||||
);
|
||||
}
|
||||
|
||||
/// Lowercase variant of an entry is still "exact" — we never nag the
|
||||
/// LLM with a hint for a token that differs only in case.
|
||||
/// Lowercase variant of an entry is still "exact" — we don't
|
||||
/// rewrite a token that differs only in case.
|
||||
#[test]
|
||||
fn noop_on_case_only_difference() {
|
||||
let g = from_entries(&["Cerebrum"]);
|
||||
assert_eq!(
|
||||
g.annotate("Patient mit Blutung im cerebrum."),
|
||||
g.replace("Patient mit Blutung im cerebrum."),
|
||||
"Patient mit Blutung im cerebrum."
|
||||
);
|
||||
}
|
||||
|
||||
/// A lowercase token that's also a spelling variant (not exact) is
|
||||
/// replaced with the vocab's canonical casing.
|
||||
#[test]
|
||||
fn replace_preserves_original_casing() {
|
||||
let g = from_entries(&["Cerebrum"]);
|
||||
let out = g.replace("Blutung im zerebrum heute.");
|
||||
assert_eq!(out, "Blutung im Cerebrum heute.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn noop_on_unrelated_token() {
|
||||
let g = from_entries(&["Cerebrum"]);
|
||||
let input = "Patient hat Husten und Fieber.";
|
||||
assert_eq!(g.annotate(input), input);
|
||||
assert_eq!(g.replace(input), input);
|
||||
}
|
||||
|
||||
/// Entries shorter than MIN_TOKEN_LEN are rejected at load time —
|
||||
@@ -241,28 +251,26 @@ mod tests {
|
||||
let g = from_entries(&["Nuck", "Bein", "Behn"]);
|
||||
assert_eq!(g.len(), 0, "short entries must be dropped");
|
||||
let input = "Patient klagt nach schwerem Heben.";
|
||||
assert_eq!(g.annotate(input), input);
|
||||
assert_eq!(g.replace(input), input);
|
||||
}
|
||||
|
||||
/// Classic ASR-substitution case (N↔D) — distance 1 match.
|
||||
#[test]
|
||||
fn annotates_consonant_substitution() {
|
||||
fn replaces_consonant_substitution() {
|
||||
let g = from_entries(&["Carvedilol"]);
|
||||
let out = g.annotate("Patient nimmt Carvenilol.");
|
||||
assert!(
|
||||
out.contains("Carvedilol [?Carvenilol]"),
|
||||
"got: {out:?}"
|
||||
assert_eq!(
|
||||
g.replace("Patient nimmt Carvenilol."),
|
||||
"Patient nimmt Carvedilol."
|
||||
);
|
||||
}
|
||||
|
||||
/// Dropped leading consonant — edit-distance 1 from Berodual.
|
||||
#[test]
|
||||
fn annotates_dropped_consonant() {
|
||||
fn replaces_dropped_consonant() {
|
||||
let g = from_entries(&["Berodual"]);
|
||||
let out = g.annotate("Patient bekommt Erodual.");
|
||||
assert!(
|
||||
out.contains("Berodual [?Erodual]"),
|
||||
"got: {out:?}"
|
||||
assert_eq!(
|
||||
g.replace("Patient bekommt Erodual."),
|
||||
"Patient bekommt Berodual."
|
||||
);
|
||||
}
|
||||
|
||||
@@ -271,10 +279,9 @@ mod tests {
|
||||
#[test]
|
||||
fn picks_closest_candidate() {
|
||||
let g = from_entries(&["Cerebrum", "Cerebrom"]);
|
||||
let out = g.annotate("Patient mit Blutung im Zerebrum.");
|
||||
assert!(
|
||||
out.contains("Cerebrum [?Zerebrum]"),
|
||||
"expected Cerebrum as closer candidate, got: {out:?}"
|
||||
assert_eq!(
|
||||
g.replace("Patient mit Blutung im Zerebrum."),
|
||||
"Patient mit Blutung im Cerebrum."
|
||||
);
|
||||
}
|
||||
|
||||
@@ -282,7 +289,7 @@ mod tests {
|
||||
fn preserves_whitespace_and_punctuation() {
|
||||
let g = from_entries(&["Cerebrum"]);
|
||||
let input = "Line 1.\n\nLine 2: foo-bar; baz!";
|
||||
assert_eq!(g.annotate(input), input);
|
||||
assert_eq!(g.replace(input), input);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
Reference in New Issue
Block a user