diff --git a/server/Cargo.lock b/server/Cargo.lock index dc33cc5..132882b 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -319,6 +319,12 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "deunicode" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" + [[package]] name = "displaydoc" version = "0.2.5" @@ -347,6 +353,7 @@ dependencies = [ "axum", "axum-extra", "bcrypt", + "deunicode", "doctate-common", "dotenvy", "filetime", diff --git a/server/Cargo.toml b/server/Cargo.toml index ca74397..a56a773 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -49,6 +49,7 @@ pulldown-cmark = { version = "0.13", default-features = false, features = ["html strsim = "0.11" spellbook = "0.4.0" subtle = "2" +deunicode = "1.6.2" [dev-dependencies] tower = { version = "0.5", features = ["util"] } diff --git a/server/src/analyze/charset.rs b/server/src/analyze/charset.rs new file mode 100644 index 0000000..3cb1aff --- /dev/null +++ b/server/src/analyze/charset.rs @@ -0,0 +1,119 @@ +//! Fold LLM output to the ISO 8859-15 representable subset. +//! +//! German practice-management systems (e.g. INDAMED MEDICAL OFFICE) store +//! free text in ISO 8859-15, an 8-bit code. When a doctor copies our cleaned +//! document and pastes it there, any codepoint outside that set renders as a +//! replacement box (□). The LLM, primed by typographic punctuation in the +//! prompt templates, emits en-/em-dashes, curly quotes, ellipses, fraction +//! slashes etc. — all absent from ISO 8859-15 — so those box out, while +//! umlauts, the micro sign `µ`, guillemets `« »` and the euro sign `€` (all +//! *within* 8859-15) paste cleanly. +//! +//! Strategy: keep every char already representable in 8859-15 verbatim, and +//! route only the residual through `deunicode`'s mature ASCII transliteration +//! table (`– → -`, `… → ...`, `½ → 1/2`, …). Because representable chars never +//! reach `deunicode`, its ASCII-only nature is harmless here — `ä`/`µ`/`€` are +//! kept, not flattened. Chars `deunicode` cannot map are left untouched (never +//! silently dropped); MO may box them, which is acceptable per design. + +use deunicode::deunicode_char; + +/// True when `c` can be encoded in ISO 8859-15. +/// +/// ISO 8859-15 is Latin-1 (ISO 8859-1) with eight byte positions reassigned: +/// `¤ ¦ ¨ ´ ¸ ¼ ½ ¾` were dropped in favour of `€ Š š Ž ž Œ œ Ÿ`. The +/// representable set is therefore printable ASCII, plus the Latin-1 high range +/// minus those eight, plus the eight replacements. Structural whitespace +/// (`\n \r \t`) is treated as representable so the markdown body survives. +pub(crate) fn is_iso8859_15(c: char) -> bool { + if matches!(c, '\n' | '\r' | '\t') { + return true; + } + match c as u32 { + // Printable ASCII. + 0x20..=0x7E => true, + // Latin-1 high range, minus the eight positions 8859-15 reassigned. + 0xA0..=0xFF => !matches!( + c, + '\u{A4}' | '\u{A6}' | '\u{A8}' | '\u{B4}' | '\u{B8}' | '\u{BC}' | '\u{BD}' | '\u{BE}' + ), + // The eight characters 8859-15 added on top of Latin-1. + _ => matches!(c, '€' | 'Š' | 'š' | 'Ž' | 'ž' | 'Œ' | 'œ' | 'Ÿ'), + } +} + +/// Rewrite `s` so every character is representable in ISO 8859-15 where a +/// sensible equivalent exists; characters with no known mapping pass through +/// unchanged. +pub(crate) fn fold_to_iso8859_15(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + if is_iso8859_15(c) { + // Already representable — keep verbatim (ä, µ, «», €, …). + out.push(c); + } else if let Some(ascii) = deunicode_char(c) { + // Residual: fold via the mature transliteration table. + out.push_str(ascii); + } else { + // No known mapping — leave it rather than drop content; MO may + // box it, which is acceptable. + out.push(c); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + // Test vectors below necessarily contain German and typographic + // characters: this module is about exactly those codepoints. + + #[test] + fn membership_matches_iso8859_15() { + // Representable: ASCII, umlauts, micro sign, guillemets, euro, etc. + for c in [ + 'a', 'Z', '9', ' ', 'ä', 'ö', 'ü', 'Ä', 'Ö', 'Ü', 'ß', 'µ', '«', '»', '€', '°', '²', + '³', '§', 'Œ', 'š', + ] { + assert!(is_iso8859_15(c), "{c:?} should be representable in 8859-15"); + } + // Not representable: typographic punctuation + the Latin-1 chars 8859-15 dropped. + for c in [ + '\u{2013}', '\u{2014}', '\u{201E}', '\u{201D}', '\u{2018}', '\u{2026}', '\u{2044}', + '\u{2022}', '\u{2032}', '\u{00BD}', '\u{00BC}', '\u{00BE}', '\u{00A4}', + ] { + assert!(!is_iso8859_15(c), "{c:?} is NOT in 8859-15"); + } + } + + #[test] + fn representable_text_passes_through_unchanged() { + // Every char here is in 8859-15, so folding must be the identity. + let keep = "Müller über 5 µg «Wert», 37 °C, m², 20 €"; + assert_eq!(fold_to_iso8859_15(keep), keep); + } + + #[test] + fn typographic_offenders_become_representable() { + // en-dash, em-dash, low/high curly quotes, ellipsis, ½, fraction slash. + let input = "3\u{2013}4 Tabletten \u{2014} \u{201E}morgens\u{201D} \u{2026} \u{00BD} Dosis \u{2044} Tag"; + let out = fold_to_iso8859_15(input); + assert!( + out.chars().all(is_iso8859_15), + "residual char not in 8859-15: {out:?}" + ); + for bad in [ + '\u{2013}', '\u{2014}', '\u{201E}', '\u{201D}', '\u{2026}', '\u{00BD}', '\u{2044}', + ] { + assert!(!out.contains(bad), "offender {bad:?} survived in {out:?}"); + } + } + + #[test] + fn en_dash_folds_to_hyphen() { + // The headline issue #13 case: en-dash between numbers. + assert_eq!(fold_to_iso8859_15("3\u{2013}4"), "3-4"); + } +} diff --git a/server/src/analyze/mod.rs b/server/src/analyze/mod.rs index aa08f96..10f63b1 100644 --- a/server/src/analyze/mod.rs +++ b/server/src/analyze/mod.rs @@ -5,6 +5,7 @@ use tokio::sync::mpsc; pub mod auto_trigger; pub mod backend; +pub mod charset; pub mod in_flight; pub mod llm; pub mod prompt; diff --git a/server/src/analyze/worker.rs b/server/src/analyze/worker.rs index 14b2227..6092783 100644 --- a/server/src/analyze/worker.rs +++ b/server/src/analyze/worker.rs @@ -140,6 +140,13 @@ async fn process( // `Amiodarone` back to `Amiodaron`). let document = vocab.replace(&document); + // Charset normalization: fold typographic Unicode the LLM emits (en-/em- + // dashes, curly quotes, ellipses, fraction slashes) down to the ISO + // 8859-15 subset that German practice software (e.g. MEDICAL OFFICE) can + // render. Without this, a copy-paste from the case page produces + // replacement boxes for those codepoints. See `analyze::charset`. + let document = super::charset::fold_to_iso8859_15(&document); + // Silent-deletion guard: schema-aware backends can return formally valid // but empty envelopes (`{"document": ""}`) when the prompt is incompatible // with the format constraint. Without this guard the worker would persist