Refactor gazetteer to replace post-AI

The gazetteer has been refactored to act as a post-AI normalization
filter. Previously, it was used to annotate LLM input with potential
corrections from a vocabulary. This approach was ineffective because
LLMs often override such hints.

The new approach applies the gazetteer *after* the LLM has generated its
output. This allows for deterministic correction of known terminology,
including fixing LLM output drift (e.g., anglicized drug names).

Key changes:
- `annotate` function renamed to `replace`.
- The output format changes from `Canonical [?original]` to simply the
  `Canonical` form.
- The gazetteer now operates on the final LLM output before persistence,
  ensuring consistency.
- Tests have been updated to reflect this new behavior, focusing on the
  final output rather than the LLM request payload.

server/src/analyze/prompt.rs

@@ -2,7 +2,7 @@ use super::AnalysisInput;
 
 /// System prompt for the consolidation LLM. From docs/projektplan.md:365-369.
 /// Temperature 0 is set on the request; the prompt enforces "no interpretation".
-pub const SYSTEM_PROMPT: &str = "Du bist ein medizinischer Assistent. Fasse die folgenden diktierten Abschnitte zu einem zusammenhängenden Text zusammen. Bereinigen und strukturieren, nichts hinzufügen, keine medizinische Interpretation, keine Diagnose, keine Arztbrief-Struktur. Entferne Redundanzen. Spätere Aufnahmen haben Vorrang — Korrekturen, Nachträge und Widersprüche zugunsten der chronologisch letzten Aussage auflösen. Antworte auf Deutsch. Nur der zusammengefasste Text, keine Einleitung, kein Abschlusssatz.\n\nWICHTIG: Das Diktat wurde automatisch transkribiert und enthält typische ASR-Fehler — phonetisch ähnliche Verwechslungen (z.B. \"Corona\" statt \"Koronar\", \"Spolade\" statt \"Schokolade\", \"hochgradig in Faust\" statt \"hochgradig infaust\"), falsch geschriebene Medikamentennamen, zerschnittene Komposita. Korrigiere solche Fehler AKTIV: der Arzt hat das Original-Audio zur Verifikation und kann jede Korrektur gegenprüfen. Bei sicherer Korrektur: schreibe das korrigierte Wort. Bei Unsicherheit, wie die Korrektur lauten müsste: behalte das Original bei.\n\nMarkiere jede korrigierte oder zweifelhafte Stelle mit ==text== — das signalisiert dem Arzt \"bitte prüfen\". Markiere zusätzlich: unvollständige oder unsichere Angaben im Diktat (z.B. \"Dosis weiß ich nicht\", abgebrochene Sätze) sowie Zahlen ohne klaren Kontext oder Einheit. Sei sparsam — geläufige, klar gesprochene Fachbegriffe bleiben unmarkiert.\n\nDer Text kann Annotationen der Form `Kandidat [?Original]` enthalten — das sind Korrekturen aus einem Fach-Wörterbuch. `Kandidat` ist die vorgeschlagene Korrektur, `Original` das, was Whisper ursprünglich transkribiert hat. Prüfe mit dem Satzkontext: passt der Kandidat, übernimm ihn und markiere mit ==. Passt er nicht, verwende stattdessen das Original ohne Markierung. Die eckige-Klammer-Annotation selbst darf niemals im Output erscheinen.";
+pub const SYSTEM_PROMPT: &str = "Du bist ein medizinischer Assistent. Fasse die folgenden diktierten Abschnitte zu einem zusammenhängenden Text zusammen. Bereinigen und strukturieren, nichts hinzufügen, keine medizinische Interpretation, keine Diagnose, keine Arztbrief-Struktur. Entferne Redundanzen. Spätere Aufnahmen haben Vorrang — Korrekturen, Nachträge und Widersprüche zugunsten der chronologisch letzten Aussage auflösen. Antworte auf Deutsch. Nur der zusammengefasste Text, keine Einleitung, kein Abschlusssatz.\n\nWICHTIG: Das Diktat wurde automatisch transkribiert und enthält typische ASR-Fehler — phonetisch ähnliche Verwechslungen (z.B. \"Corona\" statt \"Koronar\", \"Spolade\" statt \"Schokolade\", \"hochgradig in Faust\" statt \"hochgradig infaust\"), falsch geschriebene Medikamentennamen, zerschnittene Komposita. Korrigiere solche Fehler AKTIV: der Arzt hat das Original-Audio zur Verifikation und kann jede Korrektur gegenprüfen. Bei sicherer Korrektur: schreibe das korrigierte Wort. Bei Unsicherheit, wie die Korrektur lauten müsste: behalte das Original bei.\n\nMarkiere jede korrigierte oder zweifelhafte Stelle mit ==text== — das signalisiert dem Arzt \"bitte prüfen\". Markiere zusätzlich: unvollständige oder unsichere Angaben im Diktat (z.B. \"Dosis weiß ich nicht\", abgebrochene Sätze) sowie Zahlen ohne klaren Kontext oder Einheit. Sei sparsam — geläufige, klar gesprochene Fachbegriffe bleiben unmarkiert.";
 
 /// Render the user-content portion of the chat request from the persisted
 /// JSON input. Recordings with blank text are skipped — they carry no signal

server/src/analyze/worker.rs

@@ -78,12 +78,7 @@ async fn process(
         return;
     }
 
-    // Render the raw transcript bundle, then inject gazetteer hints for
-    // tokens that phonetically resemble known proper names. The LLM sees
-    // annotations of the form `Token [?Candidate]` and decides in context
-    // whether to adopt them.
     let user_content = prompt::render_prompt(&input);
-    let user_content = vocab.annotate(&user_content);
     let total_chars = user_content.chars().count();
     info!(
         case = %case_dir.display(),
@@ -120,6 +115,11 @@ async fn process(
         }
     };
 
+    // Post-AI terminology normalization: enforce vocab-canonical forms
+    // over the LLM's training priors (e.g. rewrite anglicized INNs like
+    // `Amiodarone` back to `Amiodaron`).
+    let document = vocab.replace(&document);
+
     if let Err(e) = write_atomic(&document_path, document.as_bytes()).await {
         error!(path = %document_path.display(), error = %e, "writing document failed");
         return;

server/src/gazetteer/mod.rs

@@ -1,18 +1,24 @@
-//! Gazetteer: domain-vocabulary lookup that injects correction hints
-//! before the analysis LLM runs.
+//! Gazetteer: deterministic terminology normalization filter applied
+//! to every AI output (Whisper, oneliner LLM, analysis LLM) before
+//! persistence.
 //!
-//! The LLM cannot know invented proper names (drug brands, uncommon
-//! technical terms). Whisper misrenders them, usually within a couple
-//! of edits of the real spelling. This module finds those tokens and
-//! rewrites them as `Canonical [?original]` — the vocabulary form
-//! becomes the anchor, the original Whisper token sits in the bracket
-//! as a veto option. The LLM decides in context whether to keep the
-//! canonical or revert to the original.
+//! Invariant: every AI-produced text passes through `replace()` before
+//! being written to disk or forwarded downstream. Tokens within
+//! edit-distance 2 of a curated vocabulary entry are silently rewritten
+//! to the canonical form. No hints, no annotations, no LLM veto — the
+//! vocab is authoritative.
+//!
+//! Rationale: LLMs have strong training priors that override contextual
+//! hints — e.g. they anglicize `Amiodaron` to `Amiodarone` even when
+//! the input was correct. A pre-LLM hint channel cannot fix this
+//! because the drift happens in the LLM's output. Running the filter
+//! post-AI catches both Whisper typos and LLM drift with a single
+//! mechanism.
 //!
 //! Matcher: single-stage linear Damerau-Levenshtein scan over all
 //! entries, filtered by distance ≤ MAX_EDIT_DISTANCE. At typical sizes
-//! (~200 entries × ~50 transcript tokens), the per-pass cost stays well
-//! under a millisecond — dwarfed by the downstream LLM call.
+//! (~200 entries × ~50 tokens), the per-pass cost stays well under a
+//! millisecond — negligible next to the AI call it follows.
 
 use std::collections::HashSet;
 use std::io;
@@ -40,7 +46,7 @@ pub struct Gazetteer {
     /// Canonical entries in original casing — source of truth for both
     /// scanning and emission.
     entries: Vec<Box<str>>,
-    /// Lowercased canonicals; short-circuits annotation when the token
+    /// Lowercased canonicals; short-circuits replacement when the token
     /// already matches a known entry exactly.
     exact: HashSet<Box<str>>,
 }
@@ -93,13 +99,12 @@ impl Gazetteer {
         self.entries.push(entry.into());
     }
 
-    /// Annotate `text` by replacing any word-token within edit-distance
-    /// 2 of a gazetteer entry with `Canonical [?original]`. The
-    /// canonical takes the anchor position so the LLM's default action
-    /// is to adopt it; the original Whisper token stays available in
-    /// the bracket as a veto option. Exact matches and non-word
-    /// segments pass through unchanged.
-    pub fn annotate(&self, text: &str) -> String {
+    /// Rewrite every word-token within edit-distance 2 of a gazetteer
+    /// entry to the canonical form (original casing from the vocab
+    /// file). Exact matches and non-word segments pass through
+    /// unchanged. Each replacement is logged at info level as an
+    /// audit trail.
+    pub fn replace(&self, text: &str) -> String {
         if self.is_empty() {
             return text.to_string();
         }
@@ -112,12 +117,9 @@ impl Gazetteer {
                             token = w,
                             canonical = c,
                             distance = d,
-                            "gazetteer hit"
+                            "gazetteer replaced"
                         );
                         out.push_str(c);
-                        out.push_str(" [?");
-                        out.push_str(w);
-                        out.push(']');
                     }
                     None => out.push_str(w),
                 },
@@ -194,16 +196,15 @@ mod tests {
     fn empty_gazetteer_is_pass_through() {
         let g = Gazetteer::empty();
         let input = "Patient nahm Zerebrum-Medikation.";
-        assert_eq!(g.annotate(input), input);
+        assert_eq!(g.replace(input), input);
     }
 
     #[test]
-    fn annotates_typo_within_distance() {
+    fn replaces_typo_within_distance() {
         let g = from_entries(&["Cerebrum"]);
-        let out = g.annotate("Patient mit Blutung im Zerebrum.");
-        assert!(
-            out.contains("Cerebrum [?Zerebrum]"),
-            "expected annotation, got: {out:?}"
+        assert_eq!(
+            g.replace("Patient mit Blutung im Zerebrum."),
+            "Patient mit Blutung im Cerebrum."
         );
     }
 
@@ -211,27 +212,36 @@ mod tests {
     fn noop_on_exact_match() {
         let g = from_entries(&["Cerebrum"]);
         assert_eq!(
-            g.annotate("Patient mit Blutung im Cerebrum."),
+            g.replace("Patient mit Blutung im Cerebrum."),
             "Patient mit Blutung im Cerebrum."
         );
     }
 
-    /// Lowercase variant of an entry is still "exact" — we never nag the
-    /// LLM with a hint for a token that differs only in case.
+    /// Lowercase variant of an entry is still "exact" — we don't
+    /// rewrite a token that differs only in case.
     #[test]
     fn noop_on_case_only_difference() {
         let g = from_entries(&["Cerebrum"]);
         assert_eq!(
-            g.annotate("Patient mit Blutung im cerebrum."),
+            g.replace("Patient mit Blutung im cerebrum."),
             "Patient mit Blutung im cerebrum."
         );
     }
 
+    /// A lowercase token that's also a spelling variant (not exact) is
+    /// replaced with the vocab's canonical casing.
+    #[test]
+    fn replace_preserves_original_casing() {
+        let g = from_entries(&["Cerebrum"]);
+        let out = g.replace("Blutung im zerebrum heute.");
+        assert_eq!(out, "Blutung im Cerebrum heute.");
+    }
+
     #[test]
     fn noop_on_unrelated_token() {
         let g = from_entries(&["Cerebrum"]);
         let input = "Patient hat Husten und Fieber.";
-        assert_eq!(g.annotate(input), input);
+        assert_eq!(g.replace(input), input);
     }
 
     /// Entries shorter than MIN_TOKEN_LEN are rejected at load time —
@@ -241,28 +251,26 @@ mod tests {
         let g = from_entries(&["Nuck", "Bein", "Behn"]);
         assert_eq!(g.len(), 0, "short entries must be dropped");
         let input = "Patient klagt nach schwerem Heben.";
-        assert_eq!(g.annotate(input), input);
+        assert_eq!(g.replace(input), input);
     }
 
     /// Classic ASR-substitution case (N↔D) — distance 1 match.
     #[test]
-    fn annotates_consonant_substitution() {
+    fn replaces_consonant_substitution() {
         let g = from_entries(&["Carvedilol"]);
-        let out = g.annotate("Patient nimmt Carvenilol.");
-        assert!(
-            out.contains("Carvedilol [?Carvenilol]"),
-            "got: {out:?}"
+        assert_eq!(
+            g.replace("Patient nimmt Carvenilol."),
+            "Patient nimmt Carvedilol."
         );
     }
 
     /// Dropped leading consonant — edit-distance 1 from Berodual.
     #[test]
-    fn annotates_dropped_consonant() {
+    fn replaces_dropped_consonant() {
         let g = from_entries(&["Berodual"]);
-        let out = g.annotate("Patient bekommt Erodual.");
-        assert!(
-            out.contains("Berodual [?Erodual]"),
-            "got: {out:?}"
+        assert_eq!(
+            g.replace("Patient bekommt Erodual."),
+            "Patient bekommt Berodual."
         );
     }
 
@@ -271,10 +279,9 @@ mod tests {
     #[test]
     fn picks_closest_candidate() {
         let g = from_entries(&["Cerebrum", "Cerebrom"]);
-        let out = g.annotate("Patient mit Blutung im Zerebrum.");
-        assert!(
-            out.contains("Cerebrum [?Zerebrum]"),
-            "expected Cerebrum as closer candidate, got: {out:?}"
+        assert_eq!(
+            g.replace("Patient mit Blutung im Zerebrum."),
+            "Patient mit Blutung im Cerebrum."
         );
     }
 
@@ -282,7 +289,7 @@ mod tests {
     fn preserves_whitespace_and_punctuation() {
         let g = from_entries(&["Cerebrum"]);
         let input = "Line 1.\n\nLine 2: foo-bar; baz!";
-        assert_eq!(g.annotate(input), input);
+        assert_eq!(g.replace(input), input);
     }
 
     #[tokio::test]

server/tests/analyze_test.rs

@@ -360,13 +360,13 @@ async fn analyze_worker_writes_document_via_wiremock() {
     let _ = tokio::time::timeout(Duration::from_secs(2), handle).await;
 }
 
-/// The analyze worker must route transcripts through the Gazetteer before
-/// sending them to the LLM. "Zerebrum" differs from the gazetteer entry
-/// "Cerebrum" by exactly one character — so the annotation
-/// `Cerebrum [?Zerebrum]` (canonical anchor, original in bracket) must
-/// end up in the LLM request body.
+/// The analyze worker must pass the LLM output through the Gazetteer
+/// as a post-AI normalization filter. The mock LLM is primed to return
+/// "Zerebrum" (an edit-distance-1 variant of the vocab entry
+/// "Cerebrum"). The persisted `document.md` must contain the canonical
+/// "Cerebrum", not the LLM's drift.
 #[tokio::test]
-async fn analyze_worker_annotates_with_gazetteer() {
+async fn analyze_worker_normalizes_llm_output() {
     let tmp = unique_tmp("g");
     let case_dir = tmp.join("dr_a/22222222-2222-2222-2222-222222222222");
     std::fs::create_dir_all(&case_dir).unwrap();
@@ -394,14 +394,14 @@ async fn analyze_worker_annotates_with_gazetteer() {
     );
     assert_eq!(vocab.len(), 1);
 
-    // Mock LLM: responds 200 unconditionally. The assertion is on the
-    // *request body*, not the response.
+    // Mock LLM returns the *uncorrected* form — we're proving the
+    // gazetteer rewrites it on the way out of the analyze worker.
     let mock = MockServer::start().await;
     Mock::given(method("POST"))
         .and(path("/v1/chat/completions"))
         .respond_with(ResponseTemplate::new(200).set_body_json(json!({
             "choices": [{
-                "message": { "content": "Patient mit Blutung im ==Cerebrum==." }
+                "message": { "content": "Patient mit Blutung im Zerebrum." }
             }]
         })))
         .mount(&mock)
@@ -427,14 +427,14 @@ async fn analyze_worker_annotates_with_gazetteer() {
         tokio::time::sleep(Duration::from_millis(20)).await;
     }
 
-    // Inspect what actually went to the LLM.
+    // The persisted document must show the canonical form even though
+    // the LLM returned the drift form.
+    let document = std::fs::read_to_string(&document_path).unwrap();
+    assert_eq!(document, "Patient mit Blutung im Cerebrum.");
+
+    // Sanity: one LLM call happened.
     let requests = mock.received_requests().await.unwrap();
     assert_eq!(requests.len(), 1, "expected exactly one LLM request");
-    let body = String::from_utf8(requests[0].body.clone()).unwrap();
-    assert!(
-        body.contains("Cerebrum [?Zerebrum]"),
-        "expected gazetteer annotation in LLM request body, got:\n{body}",
-    );
 
     drop(tx);
     let _ = tokio::time::timeout(Duration::from_secs(2), handle).await;
@@ -442,14 +442,14 @@ async fn analyze_worker_annotates_with_gazetteer() {
 
 /// Manual debug utility — not part of the default test set.
 /// Usage:
-///   DRY_CASE_DIR=/path/to/case cargo test --test analyze_test annotate_real_case_dry -- --ignored --nocapture
+///   DRY_CASE_DIR=/path/to/case cargo test --test analyze_test replace_real_case_dry -- --ignored --nocapture
 ///
 /// Prints each transcript in a given case dir side-by-side with its
-/// gazetteer-annotated version. Useful to eyeball false-positives /
+/// gazetteer-normalized version. Useful to eyeball false-positives /
 /// false-negatives on real cases.
 #[tokio::test]
 #[ignore]
-async fn annotate_real_case_dry() {
+async fn replace_real_case_dry() {
     let vocab_dir = std::env::var("DRY_VOCAB_DIR")
         .unwrap_or_else(|_| "/home/brummel/dev/doctate/server/vocab".into());
     let case_dir = std::env::var("DRY_CASE_DIR").unwrap_or_else(|_| {
@@ -475,12 +475,12 @@ async fn annotate_real_case_dry() {
 
     for path in names {
         let text = std::fs::read_to_string(&path).unwrap();
-        let annotated = vocab.annotate(&text);
+        let replaced = vocab.replace(&text);
         println!("\n=== {} ===", path.file_name().unwrap().to_string_lossy());
         println!("--- original ---\n{text}");
-        println!("--- annotated ---\n{annotated}");
-        if annotated == text {
-            println!("[no annotations applied]");
+        println!("--- replaced ---\n{replaced}");
+        if replaced == text {
+            println!("[no replacements applied]");
         }
     }
 }