fix(analyze): fold document output to ISO 8859-15 for Medical Office paste

German practice software (INDAMED MEDICAL OFFICE) stores free text in
ISO 8859-15, an 8-bit code. Pasting our cleaned document there rendered
typographic Unicode the LLM emits — en-/em-dashes, curly quotes,
ellipses, fraction slashes — as replacement boxes, while umlauts, the
micro sign, guillemets and the euro sign (all within 8859-15) pasted
fine. The KBV xDT data standard confirms ISO 8859-15 as the canonical
charset for German practice systems.

Add `analyze::charset::fold_to_iso8859_15`, applied in the worker right
after the gazetteer and before `content_md` is persisted: representable
chars pass through verbatim, the residual is transliterated via the
`deunicode` table (en-dash -> '-', ellipsis -> '...', '1/2' -> "1/2"),
and unmapped chars are left untouched rather than dropped. Because
representable chars never reach deunicode, its ASCII-only nature does
not flatten umlauts or the micro sign.

refs #13

server/Cargo.lock

@@ -319,6 +319,12 @@ dependencies = [
  "powerfmt",
 ]
 
+[[package]]
+name = "deunicode"
+version = "1.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04"
+
 [[package]]
 name = "displaydoc"
 version = "0.2.5"
@@ -347,6 +353,7 @@ dependencies = [
  "axum",
  "axum-extra",
  "bcrypt",
+ "deunicode",
  "doctate-common",
  "dotenvy",
  "filetime",

server/Cargo.toml

@@ -49,6 +49,7 @@ pulldown-cmark = { version = "0.13", default-features = false, features = ["html
 strsim = "0.11"
 spellbook = "0.4.0"
 subtle = "2"
+deunicode = "1.6.2"
 
 [dev-dependencies]
 tower = { version = "0.5", features = ["util"] }

server/src/analyze/charset.rs

@@ -0,0 +1,119 @@
+//! Fold LLM output to the ISO 8859-15 representable subset.
+//!
+//! German practice-management systems (e.g. INDAMED MEDICAL OFFICE) store
+//! free text in ISO 8859-15, an 8-bit code. When a doctor copies our cleaned
+//! document and pastes it there, any codepoint outside that set renders as a
+//! replacement box (□). The LLM, primed by typographic punctuation in the
+//! prompt templates, emits en-/em-dashes, curly quotes, ellipses, fraction
+//! slashes etc. — all absent from ISO 8859-15 — so those box out, while
+//! umlauts, the micro sign `µ`, guillemets `« »` and the euro sign `€` (all
+//! *within* 8859-15) paste cleanly.
+//!
+//! Strategy: keep every char already representable in 8859-15 verbatim, and
+//! route only the residual through `deunicode`'s mature ASCII transliteration
+//! table (`– → -`, `… → ...`, `½ → 1/2`, …). Because representable chars never
+//! reach `deunicode`, its ASCII-only nature is harmless here — `ä`/`µ`/`€` are
+//! kept, not flattened. Chars `deunicode` cannot map are left untouched (never
+//! silently dropped); MO may box them, which is acceptable per design.
+
+use deunicode::deunicode_char;
+
+/// True when `c` can be encoded in ISO 8859-15.
+///
+/// ISO 8859-15 is Latin-1 (ISO 8859-1) with eight byte positions reassigned:
+/// `¤ ¦ ¨ ´ ¸ ¼ ½ ¾` were dropped in favour of `€ Š š Ž ž Œ œ Ÿ`. The
+/// representable set is therefore printable ASCII, plus the Latin-1 high range
+/// minus those eight, plus the eight replacements. Structural whitespace
+/// (`\n \r \t`) is treated as representable so the markdown body survives.
+pub(crate) fn is_iso8859_15(c: char) -> bool {
+    if matches!(c, '\n' | '\r' | '\t') {
+        return true;
+    }
+    match c as u32 {
+        // Printable ASCII.
+        0x20..=0x7E => true,
+        // Latin-1 high range, minus the eight positions 8859-15 reassigned.
+        0xA0..=0xFF => !matches!(
+            c,
+            '\u{A4}' | '\u{A6}' | '\u{A8}' | '\u{B4}' | '\u{B8}' | '\u{BC}' | '\u{BD}' | '\u{BE}'
+        ),
+        // The eight characters 8859-15 added on top of Latin-1.
+        _ => matches!(c, '€' | 'Š' | 'š' | 'Ž' | 'ž' | 'Œ' | 'œ' | 'Ÿ'),
+    }
+}
+
+/// Rewrite `s` so every character is representable in ISO 8859-15 where a
+/// sensible equivalent exists; characters with no known mapping pass through
+/// unchanged.
+pub(crate) fn fold_to_iso8859_15(s: &str) -> String {
+    let mut out = String::with_capacity(s.len());
+    for c in s.chars() {
+        if is_iso8859_15(c) {
+            // Already representable — keep verbatim (ä, µ, «», €, …).
+            out.push(c);
+        } else if let Some(ascii) = deunicode_char(c) {
+            // Residual: fold via the mature transliteration table.
+            out.push_str(ascii);
+        } else {
+            // No known mapping — leave it rather than drop content; MO may
+            // box it, which is acceptable.
+            out.push(c);
+        }
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // Test vectors below necessarily contain German and typographic
+    // characters: this module is about exactly those codepoints.
+
+    #[test]
+    fn membership_matches_iso8859_15() {
+        // Representable: ASCII, umlauts, micro sign, guillemets, euro, etc.
+        for c in [
+            'a', 'Z', '9', ' ', 'ä', 'ö', 'ü', 'Ä', 'Ö', 'Ü', 'ß', 'µ', '«', '»', '€', '°', '²',
+            '³', '§', 'Œ', 'š',
+        ] {
+            assert!(is_iso8859_15(c), "{c:?} should be representable in 8859-15");
+        }
+        // Not representable: typographic punctuation + the Latin-1 chars 8859-15 dropped.
+        for c in [
+            '\u{2013}', '\u{2014}', '\u{201E}', '\u{201D}', '\u{2018}', '\u{2026}', '\u{2044}',
+            '\u{2022}', '\u{2032}', '\u{00BD}', '\u{00BC}', '\u{00BE}', '\u{00A4}',
+        ] {
+            assert!(!is_iso8859_15(c), "{c:?} is NOT in 8859-15");
+        }
+    }
+
+    #[test]
+    fn representable_text_passes_through_unchanged() {
+        // Every char here is in 8859-15, so folding must be the identity.
+        let keep = "Müller über 5 µg «Wert», 37 °C, m², 20 €";
+        assert_eq!(fold_to_iso8859_15(keep), keep);
+    }
+
+    #[test]
+    fn typographic_offenders_become_representable() {
+        // en-dash, em-dash, low/high curly quotes, ellipsis, ½, fraction slash.
+        let input = "3\u{2013}4 Tabletten \u{2014} \u{201E}morgens\u{201D} \u{2026} \u{00BD} Dosis \u{2044} Tag";
+        let out = fold_to_iso8859_15(input);
+        assert!(
+            out.chars().all(is_iso8859_15),
+            "residual char not in 8859-15: {out:?}"
+        );
+        for bad in [
+            '\u{2013}', '\u{2014}', '\u{201E}', '\u{201D}', '\u{2026}', '\u{00BD}', '\u{2044}',
+        ] {
+            assert!(!out.contains(bad), "offender {bad:?} survived in {out:?}");
+        }
+    }
+
+    #[test]
+    fn en_dash_folds_to_hyphen() {
+        // The headline issue #13 case: en-dash between numbers.
+        assert_eq!(fold_to_iso8859_15("3\u{2013}4"), "3-4");
+    }
+}

server/src/analyze/mod.rs

@@ -5,6 +5,7 @@ use tokio::sync::mpsc;
 
 pub mod auto_trigger;
 pub mod backend;
+pub mod charset;
 pub mod in_flight;
 pub mod llm;
 pub mod prompt;

server/src/analyze/worker.rs

@@ -140,6 +140,13 @@ async fn process(
     // `Amiodarone` back to `Amiodaron`).
     let document = vocab.replace(&document);
 
+    // Charset normalization: fold typographic Unicode the LLM emits (en-/em-
+    // dashes, curly quotes, ellipses, fraction slashes) down to the ISO
+    // 8859-15 subset that German practice software (e.g. MEDICAL OFFICE) can
+    // render. Without this, a copy-paste from the case page produces
+    // replacement boxes for those codepoints. See `analyze::charset`.
+    let document = super::charset::fold_to_iso8859_15(&document);
+
     // Silent-deletion guard: schema-aware backends can return formally valid
     // but empty envelopes (`{"document": ""}`) when the prompt is incompatible
     // with the format constraint. Without this guard the worker would persist