diff --git a/.gitignore b/.gitignore index ce93c33..17b974d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,3 @@ users.toml *.snap.new repomix-output.xml tmpdata/ -server/vocab/raw/ diff --git a/server/scripts/build-anatomy.sh b/server/scripts/build-anatomy.sh deleted file mode 100755 index 9a085dc..0000000 --- a/server/scripts/build-anatomy.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# -# Build vocab/anatomy.txt from scratch: -# 1. Crawl de.wikipedia.org Kategorie:Anatomie → raw/anatomy-wiki.txt -# 2. Filter against hunspell-de → vocab/anatomy.txt -# -# Idempotent: safe to re-run. Commit vocab/anatomy.txt after a successful -# build so the server uses a deterministic artefact, not a live crawl. -# -# Usage: (from anywhere) bash path/to/build-anatomy.sh - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -VOCAB_DIR="$(cd "$SCRIPT_DIR/../vocab" && pwd)" -RAW_DIR="$VOCAB_DIR/raw" - -mkdir -p "$RAW_DIR" - -RAW_FILE="$RAW_DIR/anatomy-wiki.txt" -OUT_FILE="$VOCAB_DIR/anatomy.txt" - -echo "[1/2] Fetching Wikipedia-de Kategorie:Anatomie..." -"$SCRIPT_DIR/fetch-wiki-anatomy.sh" > "$RAW_FILE" -echo " raw titles: $(wc -l < "$RAW_FILE")" - -echo "[2/2] Filtering against hunspell-de..." -"$SCRIPT_DIR/filter-hunspell-de.sh" "$RAW_FILE" "$OUT_FILE" -echo " kept unknown: $(wc -l < "$OUT_FILE")" - -echo "Done. Updated: $OUT_FILE" diff --git a/server/scripts/fetch-wiki-anatomy.sh b/server/scripts/fetch-wiki-anatomy.sh deleted file mode 100755 index e5ed4b8..0000000 --- a/server/scripts/fetch-wiki-anatomy.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash -# -# Crawl the German Wikipedia "Kategorie:Anatomie" (recursively into -# subcategories) and print each page title on stdout, one per line, -# sorted and deduplicated. Progress info goes to stderr. -# -# Dependencies: curl, jq. -# Rate limit: ~10 requests/sec (sleep 0.1 between requests). -# Depth limit: MAX_DEPTH — prevents infinite recursion via cross-links. -# -# Usage: ./fetch-wiki-anatomy.sh > raw/anatomy-wiki.txt - -set -euo pipefail - -USER_AGENT="doctate-vocab-builder (mould73@gmail.com)" -API="https://de.wikipedia.org/w/api.php" -ROOT_CATEGORY="Kategorie:Anatomie" -MAX_DEPTH=4 - -for tool in curl jq; do - if ! command -v "$tool" >/dev/null 2>&1; then - echo "Error: $tool is required but not installed." >&2 - exit 1 - fi -done - -PAGES=$(mktemp) -SEEN=$(mktemp) -trap 'rm -f "$PAGES" "$SEEN"' EXIT - -# Fetch all pages under one category, recursing into subcategories. -# Args: category_title depth -fetch_category() { - local cat="$1" - local depth="$2" - - if [[ $depth -gt $MAX_DEPTH ]]; then - return - fi - if grep -qFx "$cat" "$SEEN"; then - return - fi - echo "$cat" >> "$SEEN" - - echo "[depth=$depth] $cat" >&2 - - local cmcontinue="" - while true; do - local response - if [[ -n "$cmcontinue" ]]; then - response=$(curl -sS -A "$USER_AGENT" -G "$API" \ - --data-urlencode "action=query" \ - --data-urlencode "list=categorymembers" \ - --data-urlencode "cmtitle=$cat" \ - --data-urlencode "cmlimit=500" \ - --data-urlencode "cmtype=page|subcat" \ - --data-urlencode "format=json" \ - --data-urlencode "cmcontinue=$cmcontinue") - else - response=$(curl -sS -A "$USER_AGENT" -G "$API" \ - --data-urlencode "action=query" \ - --data-urlencode "list=categorymembers" \ - --data-urlencode "cmtitle=$cat" \ - --data-urlencode "cmlimit=500" \ - --data-urlencode "cmtype=page|subcat" \ - --data-urlencode "format=json") - fi - - # Collect page titles (ns=0 is "Main" namespace, i.e. articles). - echo "$response" | jq -r '.query.categorymembers[] | select(.ns == 0) | .title' >> "$PAGES" - - # Recurse into subcategories (ns=14). - local subcats - subcats=$(echo "$response" | jq -r '.query.categorymembers[] | select(.ns == 14) | .title') - while IFS= read -r subcat; do - [[ -z "$subcat" ]] && continue - fetch_category "$subcat" $((depth + 1)) - done <<< "$subcats" - - cmcontinue=$(echo "$response" | jq -r '.continue.cmcontinue // empty') - if [[ -z "$cmcontinue" ]]; then - break - fi - - sleep 0.1 - done -} - -fetch_category "$ROOT_CATEGORY" 0 - -echo "Collected $(wc -l < "$PAGES") raw titles, emitting sorted-unique..." >&2 -sort -u "$PAGES" diff --git a/server/scripts/filter-hunspell-de.sh b/server/scripts/filter-hunspell-de.sh deleted file mode 100755 index b7e4c9c..0000000 --- a/server/scripts/filter-hunspell-de.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# -# Filter a word list to entries NOT recognised by hunspell-de. Uses -# `hunspell -l` (list-unknown-words mode) — a single batch invocation, -# orders of magnitude faster than per-word lookups. -# -# Multi-word input lines are split into words by hunspell; only the -# individual unknown words are kept. That matches our gazetteer's -# token-level matching model (see gazetteer/mod.rs). -# -# Dependencies: hunspell + German dictionary. Package names vary: -# Arch/CachyOS: pacman -S hunspell hunspell-de -# Debian/Ubuntu: apt install hunspell hunspell-de-de -# Fedora: dnf install hunspell hunspell-de -# -# Usage: ./filter-hunspell-de.sh - -set -euo pipefail - -if [[ $# -ne 2 ]]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -INPUT="$1" -OUTPUT="$2" - -# Minimum entry length. Must match MIN_TOKEN_LEN in src/gazetteer/mod.rs. -# 5 is the current value — appropriate for curated medication lists where -# the 4-char eponym-vs-German-word collision class is not present. -MIN_LEN=5 - -if ! command -v hunspell >/dev/null 2>&1; then - echo "Error: hunspell is not installed. Install it via your package" >&2 - echo " manager (e.g. pacman -S hunspell, apt install hunspell)." >&2 - exit 1 -fi - -# Sanity-check: is the de_DE dictionary available? -if ! echo "Haus" | hunspell -d de_DE -a >/dev/null 2>&1; then - echo "Error: hunspell de_DE dictionary not available. Install:" >&2 - echo " Arch/CachyOS: pacman -S hunspell-de" >&2 - echo " Debian/Ubuntu: apt install hunspell-de-de" >&2 - echo " Fedora: dnf install hunspell-de" >&2 - exit 1 -fi - -# `hunspell -l` prints every unknown word from stdin, one per line. -# Drop words shorter than MIN_LEN, then sort -u for dedup + stable order. -hunspell -d de_DE -l < "$INPUT" \ - | awk -v min="$MIN_LEN" 'length($0) >= min' \ - | sort -u > "$OUTPUT" diff --git a/server/src/gazetteer/koelner.rs b/server/src/gazetteer/koelner.rs deleted file mode 100644 index b259825..0000000 --- a/server/src/gazetteer/koelner.rs +++ /dev/null @@ -1,238 +0,0 @@ -//! Kölner Phonetik (Postel 1969) — deterministic sound-hash for German -//! words. Two inputs with the same code are phonetically similar. -//! -//! We use this to find gazetteer entries that *sound like* a -//! misspelled token in the transcript. Example: both `Vomex` (real -//! drug name) and `Womax` (Whisper hallucination) encode to `3648`, -//! so a gazetteer containing `Vomex` can surface it as a candidate -//! correction for `Womax`. -//! -//! Pure function, no I/O, no allocation beyond the returned `String`. - -/// Encode `s` as a Kölner-Phonetik code. Output is a sequence of ASCII -/// digits `0`–`8`. Non-alphabetic input chars are silently dropped; -/// case is ignored. Empty input returns an empty string. -/// -/// Rules (condensed from Postel 1969 / Wikipedia): -/// - Vowels `A E I J O U Y` → `0`; leading `0` is kept, internal / -/// trailing `0`s are removed in post-processing. -/// - `H` is not encoded at all (neither emitted nor collapsed). -/// - `B` → `1`; `P` → `1`, but `PH` → `3`. -/// - `D T` → `2`, but before `C S Z` → `8`. -/// - `F V W` → `3`. -/// - `G K Q` → `4`. -/// - `C` is context-dependent: -/// * initial + before `A H K L O Q R U X` → `4` -/// * initial otherwise → `8` -/// * after `S Z` → `8` -/// * before `A H K O Q U X` → `4` -/// * otherwise → `8` -/// - `X` → `48`, but after `C K Q` → `8` (those already encoded the 4). -/// - `L` → `5`; `M N` → `6`; `R` → `7`; `S Z` → `8`. -/// - Post: collapse consecutive equal digits, then drop every non-leading `0`. -pub fn encode(s: &str) -> String { - let prepared = prepare(s); - if prepared.is_empty() { - return String::new(); - } - let raw = raw_encode(&prepared); - postprocess(&raw) -} - -/// Expand umlauts + `ß`, uppercase everything else, drop non-A-Z chars. -/// Returns a `Vec` because subsequent passes need random access -/// and look-ahead, which is awkward on `&str` (UTF-8 byte indices ≠ -/// char positions). -fn prepare(s: &str) -> Vec { - let mut out: Vec = Vec::with_capacity(s.len()); - for c in s.chars() { - match c { - 'ä' | 'Ä' => out.push('A'), - 'ö' | 'Ö' => out.push('O'), - 'ü' | 'Ü' => out.push('U'), - 'ß' => { - out.push('S'); - out.push('S'); - } - _ => { - for u in c.to_uppercase() { - if u.is_ascii_uppercase() { - out.push(u); - } - } - } - } - } - out -} - -/// Per-character encoding with one-char look-ahead and look-behind. -/// Emits ASCII digit chars into the output buffer. `H` emits nothing; -/// `X` may emit two chars. -fn raw_encode(chars: &[char]) -> Vec { - let mut out: Vec = Vec::with_capacity(chars.len() + 2); - for (i, &c) in chars.iter().enumerate() { - let next = chars.get(i + 1).copied(); - let prev = if i > 0 { chars.get(i - 1).copied() } else { None }; - match c { - 'A' | 'E' | 'I' | 'J' | 'O' | 'U' | 'Y' => out.push('0'), - 'H' => {} // not encoded - 'B' => out.push('1'), - 'P' => out.push(if next == Some('H') { '3' } else { '1' }), - 'D' | 'T' => out.push( - if matches!(next, Some('C') | Some('S') | Some('Z')) { - '8' - } else { - '2' - }, - ), - 'F' | 'V' | 'W' => out.push('3'), - 'G' | 'K' | 'Q' => out.push('4'), - 'C' => { - let d = if i == 0 { - // Word-initial C. - if matches!( - next, - Some('A') | Some('H') | Some('K') | Some('L') - | Some('O') | Some('Q') | Some('R') - | Some('U') | Some('X') - ) { - '4' - } else { - '8' - } - } else if matches!(prev, Some('S') | Some('Z')) { - // Override: after S/Z always 8, regardless of next. - '8' - } else if matches!( - next, - Some('A') | Some('H') | Some('K') | Some('O') - | Some('Q') | Some('U') | Some('X') - ) { - '4' - } else { - '8' - }; - out.push(d); - } - 'X' => { - if matches!(prev, Some('C') | Some('K') | Some('Q')) { - // Previous char already emitted the '4'; only the '8' is new. - out.push('8'); - } else { - out.push('4'); - out.push('8'); - } - } - 'L' => out.push('5'), - 'M' | 'N' => out.push('6'), - 'R' => out.push('7'), - 'S' | 'Z' => out.push('8'), - _ => {} // unreachable after prepare(), but harmless. - } - } - out -} - -/// Collapse runs of identical digits into one, then drop every internal -/// or trailing `0`. A leading `0` is preserved because it carries -/// information — the word started with a vowel. -fn postprocess(raw: &[char]) -> String { - let mut dedup: Vec = Vec::with_capacity(raw.len()); - for &c in raw { - if dedup.last() != Some(&c) { - dedup.push(c); - } - } - let mut out = String::with_capacity(dedup.len()); - for (i, &c) in dedup.iter().enumerate() { - if c == '0' && i > 0 { - continue; - } - out.push(c); - } - out -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn encodes_classic_german_surnames() { - // Reference values from the German Wikipedia article on Kölner Phonetik. - assert_eq!(encode("Müller"), "657"); - assert_eq!(encode("Meier"), "67"); - assert_eq!(encode("Meyer"), "67"); - assert_eq!(encode("Mayer"), "67"); - assert_eq!(encode("Schmidt"), "862"); - assert_eq!(encode("Schmitz"), "868"); - } - - #[test] - fn treats_umlaut_and_digraph_as_equivalent() { - // Classic test: "ü" and "ue" should normalise to the same code. - assert_eq!(encode("Müller"), encode("Mueller")); - } - - /// Regression guard for the ASR-correction use case. Womax (Whisper - /// hallucination) and Vomex (real drug name) must encode identically - /// — that is the mechanism that makes the gazetteer lookup fire. - #[test] - fn vomex_and_womax_collide() { - assert_eq!(encode("Vomex"), encode("Womax")); - assert_eq!(encode("Vomex"), "3648"); - } - - #[test] - fn empty_and_non_alphabetic_input_yields_empty_code() { - assert_eq!(encode(""), ""); - assert_eq!(encode("123 !?"), ""); - } - - #[test] - fn is_case_insensitive() { - assert_eq!(encode("MÜLLER"), encode("müller")); - assert_eq!(encode("MüLLeR"), encode("Müller")); - } - - #[test] - fn leading_vowel_keeps_leading_zero() { - // "Aber" → A=0, B=1, E=0, R=7 → "0107" → dedupe: "0107" → drop non-leading 0: "017". - assert_eq!(encode("Aber"), "017"); - } - - #[test] - fn c_before_h_at_word_start_codes_4() { - // "Christian": C(initial,next=H→4) H(skip) R=7 I=0 S=8 T(next=I,not CSZ→2) I=0 A=0 N=6 - // raw = 4,7,0,8,2,0,0,6 → dedupe = 4,7,0,8,2,0,6 → drop internal 0 = 4,7,8,2,6 - assert_eq!(encode("Christian"), "47826"); - } - - #[test] - fn x_after_c_is_single_digit() { - // "Axel": A=0, X (prev=A, emits 4+8), E=0, L=5 → 0,4,8,0,5 → dedupe same → "0485" - assert_eq!(encode("Axel"), "0485"); - // "Knixel": K=4, N=6, I=0, X (prev=I, emits 4+8), E=0, L=5 → 4,6,0,4,8,0,5 → "46485" - assert_eq!(encode("Knixel"), "46485"); - } - - #[test] - fn ignores_embedded_digits_and_punctuation() { - // "F1-Chip" → prepared: F,C,H,I,P. F=3, C(non-initial,prev=F,next=H→4), - // H(skip), I=0, P(no next=H→1). Raw: 3,4,0,1 → no dedupe → "341". - assert_eq!(encode("F1-Chip"), "341"); - } - - #[test] - fn ph_at_start_codes_3() { - // "Phil" → P(next=H→3), H(skip), I=0, L=5 → 3,0,5 → "35". - assert_eq!(encode("Phil"), "35"); - } - - #[test] - fn s_z_before_c_yields_8() { - // "Katze" → K=4, A=0, T(next=Z→8), Z=8, E=0 → 4,0,8,8,0 → dedupe 4,0,8,0 → "48". - assert_eq!(encode("Katze"), "48"); - } -} diff --git a/server/src/gazetteer/mod.rs b/server/src/gazetteer/mod.rs index 03376f7..c5ddea9 100644 --- a/server/src/gazetteer/mod.rs +++ b/server/src/gazetteer/mod.rs @@ -1,32 +1,33 @@ //! Gazetteer: domain-vocabulary lookup that injects correction hints -//! before the analysis LLM runs. See -//! `~/.claude/plans/happy-mapping-snail.md` for the design document. +//! before the analysis LLM runs. //! -//! The LLM cannot know invented proper names (drug brands, anatomical -//! Latin/Greek terms). Whisper misrenders them phonetically -//! (`Vomex` → `Womax`). This module exposes a static lookup that, given -//! a transcript, finds tokens that are likely typos of a known entry -//! and rewrites them as `token [?Canonical]`. The LLM decides in context +//! The LLM cannot know invented proper names (drug brands, uncommon +//! technical terms). Whisper misrenders them, usually within a couple +//! of edits of the real spelling. This module finds those tokens and +//! rewrites them as `token [?Canonical]`. The LLM decides in context //! whether to adopt the hint. +//! +//! Matcher: single-stage linear Damerau-Levenshtein scan over all +//! entries, filtered by distance ≤ MAX_EDIT_DISTANCE. At typical sizes +//! (~200 entries × ~50 transcript tokens), the per-pass cost stays well +//! under a millisecond — dwarfed by the downstream LLM call. -pub mod koelner; - -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::io; use std::path::Path; use strsim::damerau_levenshtein; /// Minimum char count for *both* gazetteer entries and transcript tokens. -/// Calibrated against curated medication lists: 5 keeps prototypical -/// 5-char brand names (Vomex, Tavor, Lasix, Nasic) while still blocking -/// the 4-char deutsche Alltagswort ↔ eponym collisions that unkurated -/// crawls produce (Bein↔Behn, nach↔Nuck). +/// Calibrated against curated vocabulary: 5 keeps prototypical 5-char +/// brand names (Vomex, Tavor, Lasix) while still blocking 4-char +/// collisions between everyday German words and short proper nouns +/// (Bein↔Behn, nach↔Nuck). const MIN_TOKEN_LEN: usize = 5; -/// Maximum Damerau-Levenshtein distance for a candidate to qualify as -/// a typo of a gazetteer entry. Distance 0 is excluded — the `exact` -/// set short-circuits that case. +/// Maximum Damerau-Levenshtein distance for a candidate to qualify as a +/// typo of a gazetteer entry. Distance 0 is handled by the `exact` +/// short-circuit; distances > 2 are rejected as too speculative. const MAX_EDIT_DISTANCE: usize = 2; /// Domain-specific vocabulary, populated once at server start from @@ -34,23 +35,22 @@ const MAX_EDIT_DISTANCE: usize = 2; /// via `Arc`. #[derive(Debug, Default)] pub struct Gazetteer { - /// Kölner-Phonetik code → canonical entries sharing that code. The - /// canonicals keep their original casing for display. - by_phonetic: HashMap>>, + /// Canonical entries in original casing — source of truth for both + /// scanning and emission. + entries: Vec>, /// Lowercased canonicals; short-circuits annotation when the token - /// is already spelled correctly. + /// already matches a known entry exactly. exact: HashSet>, } impl Gazetteer { - /// Empty gazetteer. `annotate` is a pure pass-through on this instance. pub fn empty() -> Self { Self::default() } /// Load every `*.txt` file (non-recursive) under `dir`. Blank lines /// and lines beginning with `#` are skipped. Case-insensitive - /// duplicates are silently collapsed. + /// duplicates across files are silently collapsed. pub async fn load_dir(dir: &Path) -> io::Result { let mut g = Self::default(); let mut entries = tokio::fs::read_dir(dir).await?; @@ -71,40 +71,29 @@ impl Gazetteer { Ok(g) } - /// Number of unique entries (case-insensitive). pub fn len(&self) -> usize { - self.exact.len() + self.entries.len() } pub fn is_empty(&self) -> bool { - self.exact.is_empty() + self.entries.is_empty() } fn insert(&mut self, entry: &str) { - // Skip short entries — they cause noisy phonetic collisions with - // ordinary German words. Same threshold as token matching. if entry.chars().count() < MIN_TOKEN_LEN { return; } let lower = entry.to_lowercase(); if !self.exact.insert(lower.into_boxed_str()) { - // Duplicate (case-insensitive) — don't add to phonetic index either. + // Duplicate (case-insensitive) — already indexed. return; } - let code = koelner::encode(entry); - if code.is_empty() { - return; - } - self.by_phonetic - .entry(code) - .or_default() - .push(entry.into()); + self.entries.push(entry.into()); } /// Annotate `text` by appending ` [?Canonical]` after any word-token - /// that looks like a misspelling of a gazetteer entry. Non-matching - /// tokens and all non-word segments (whitespace, punctuation, digits) - /// pass through byte-identical. + /// that's within edit-distance 2 of a gazetteer entry (and not an + /// exact match). Non-word segments pass through byte-identical. pub fn annotate(&self, text: &str) -> String { if self.is_empty() { return text.to_string(); @@ -136,45 +125,15 @@ impl Gazetteer { return None; } - // Stage 1: phonetic-indexed lookup. Fast, filters out the vast - // majority of non-matches in O(1). Catches same-code neighbours - // like Klexane↔Clexane, Diacepam↔Diazepam. - let code = koelner::encode(token); - if !code.is_empty() - && let Some(candidates) = self.by_phonetic.get(&code) - && let Some(hit) = best_candidate(&lower, candidates.iter().map(|c| c.as_ref())) - { - return Some(hit); - } - - // Stage 2: brute-force edit-distance over every entry. Catches - // ASR errors where Whisper substitutes phonetically-distinct - // consonants (Carvenilol↔Carvedilol: N↔D) or drops a letter - // that shifts the Kölner code (Acoxia↔Arcoxia: missing R). - // O(n) per unknown token; at n≈200 entries and ~50 tokens per - // transcript the absolute cost stays well under a millisecond. - best_candidate( - &lower, - self.by_phonetic.values().flatten().map(|c| c.as_ref()), - ) + self.entries + .iter() + .map(|c| (damerau_levenshtein(&lower, &c.to_lowercase()), c.as_ref())) + .filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE) + .min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(b.1))) + .map(|(_, c)| c) } } -/// Select the canonical entry from `candidates` that is closest to -/// `lower_token` by Damerau-Levenshtein distance, within the allowed -/// threshold. Distance 0 is rejected because that's already handled by -/// the `exact` short-circuit. Ties break alphabetically for determinism. -fn best_candidate<'a>( - lower_token: &str, - candidates: impl Iterator, -) -> Option<&'a str> { - candidates - .map(|c| (damerau_levenshtein(lower_token, &c.to_lowercase()), c)) - .filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE) - .min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(b.1))) - .map(|(_, c)| c) -} - /// A slice of the input text — either a word or a non-word run. enum Segment<'a> { Word(&'a str), @@ -212,8 +171,6 @@ mod tests { use super::*; use tempfile::tempdir; - /// Build a gazetteer from an in-memory list of canonical entries — - /// avoids filesystem I/O in the matcher tests. fn from_entries(entries: &[&str]) -> Gazetteer { let mut g = Gazetteer::empty(); for e in entries { @@ -229,12 +186,8 @@ mod tests { assert_eq!(g.annotate(input), input); } - /// Core regression: a token that's a phonetic neighbor at edit - /// distance ≤ 2 gets a hint appended. Cerebrum and Zerebrum both - /// encode to Kölner `87176` and differ by one char — a plausible - /// Whisper-style transcription slip. #[test] - fn annotates_typo_for_phonetic_neighbor() { + fn annotates_typo_within_distance() { let g = from_entries(&["Cerebrum"]); let out = g.annotate("Patient mit Blutung im Zerebrum."); assert!( @@ -248,18 +201,18 @@ mod tests { let g = from_entries(&["Cerebrum"]); assert_eq!( g.annotate("Patient mit Blutung im Cerebrum."), - "Patient mit Blutung im Cerebrum.", + "Patient mit Blutung im Cerebrum." ); } - /// Lowercase variant of an entry is still "exact" — we never nag - /// the LLM with a hint for a token that's only case-different. + /// Lowercase variant of an entry is still "exact" — we never nag the + /// LLM with a hint for a token that differs only in case. #[test] fn noop_on_case_only_difference() { let g = from_entries(&["Cerebrum"]); assert_eq!( g.annotate("Patient mit Blutung im cerebrum."), - "Patient mit Blutung im cerebrum.", + "Patient mit Blutung im cerebrum." ); } @@ -270,51 +223,42 @@ mod tests { assert_eq!(g.annotate(input), input); } - /// Gazetteer entries shorter than MIN_TOKEN_LEN are rejected at load - /// time. This prevents 4-char eponyms (Nuck, Behn, Bell, …) from - /// generating noisy false-positive matches against everyday German - /// words — the empirical reason MIN_TOKEN_LEN exists. + /// Entries shorter than MIN_TOKEN_LEN are rejected at load time — + /// prevents short-proper-noun collisions with everyday German. #[test] - fn rejects_gazetteer_entries_below_min_length() { + fn rejects_entries_below_min_length() { let g = from_entries(&["Nuck", "Bein", "Behn"]); assert_eq!(g.len(), 0, "short entries must be dropped"); - // Input with a matching short word passes through untouched. let input = "Patient klagt nach schwerem Heben."; assert_eq!(g.annotate(input), input); } - /// Edit-distance fallback: Whisper substitutes phonetically-distinct - /// consonants (here N↔D in Carvenilol↔Carvedilol). The Kölner codes - /// differ (N=6 vs D=2), so stage-1 misses — the brute-force stage - /// must catch the distance-1 match. + /// Classic ASR-substitution case (N↔D) — distance 1 match. #[test] - fn edit_distance_fallback_catches_consonant_substitution() { + fn annotates_consonant_substitution() { let g = from_entries(&["Carvedilol"]); let out = g.annotate("Patient nimmt Carvenilol."); assert!( out.contains("Carvenilol [?Carvedilol]"), - "expected fallback annotation, got: {out:?}" + "got: {out:?}" ); } - /// Edit-distance fallback: Whisper drops a consonant (leading B in - /// Berodual). Kölner codes are completely different — only the - /// linear edit-distance scan saves the match. + /// Dropped leading consonant — edit-distance 1 from Berodual. #[test] - fn edit_distance_fallback_catches_dropped_consonant() { + fn annotates_dropped_consonant() { let g = from_entries(&["Berodual"]); let out = g.annotate("Patient bekommt Erodual."); assert!( out.contains("Erodual [?Berodual]"), - "expected fallback annotation, got: {out:?}" + "got: {out:?}" ); } - /// With two phonetically-colliding candidates, the one with smaller - /// Damerau-Levenshtein distance wins. Zerebrum↔Cerebrum = 1, - /// Zerebrum↔Cerebrom = 2 → Cerebrum is chosen. + /// With multiple in-range candidates, the closest by + /// Damerau-Levenshtein wins; ties break alphabetically. #[test] - fn picks_closest_candidate_by_edit_distance() { + fn picks_closest_candidate() { let g = from_entries(&["Cerebrum", "Cerebrom"]); let out = g.annotate("Patient mit Blutung im Zerebrum."); assert!( @@ -336,7 +280,7 @@ mod tests { tokio::fs::write(dir.path().join("meds.txt"), "Aspirin\nIbuprofen\n") .await .unwrap(); - tokio::fs::write(dir.path().join("anatomy.txt"), "Coracoideus\n") + tokio::fs::write(dir.path().join("more.txt"), "Coracoideus\n") .await .unwrap(); let g = Gazetteer::load_dir(dir.path()).await.unwrap(); @@ -348,7 +292,7 @@ mod tests { let dir = tempdir().unwrap(); tokio::fs::write( dir.path().join("v.txt"), - "# medications\nAspirin\n\n \n# another\nIbuprofen\n", + "# meds\nAspirin\n\n \n# another\nIbuprofen\n", ) .await .unwrap(); diff --git a/server/vocab/README.md b/server/vocab/README.md index e25a618..b4e2d51 100644 --- a/server/vocab/README.md +++ b/server/vocab/README.md @@ -1,35 +1,22 @@ -# Gazetteer vocabulary files +# Gazetteer vocabulary -These `*.txt` files are loaded once at server startup and used to annotate -Whisper transcripts with correction hints of the form `Token [?Canonical]` -before the analysis LLM runs. Design: `~/.claude/plans/happy-mapping-snail.md`. +Plain list of domain-specific words the analysis LLM might not know — +medication brand names, proper nouns, colleagues, technical terms. Loaded +once at server start. Used to annotate close edit-distance neighbours in +Whisper transcripts as `Token [?Canonical]` for LLM-side review. -## File format +## Format -- One entry per line, UTF-8, no surrounding whitespace required. -- Lines that are blank or start with `#` are skipped. -- Case-insensitive. Duplicates across files are collapsed. -- No header row, no columns — this is a plain word list. +- One entry per line, UTF-8. +- Blank lines and lines starting with `#` are skipped. +- Matching is case-insensitive; original casing is preserved for display. +- Entries must be ≥5 characters (shorter tokens collide too often with + everyday German words). +- Duplicates (case-insensitive, across files) are collapsed automatically. -## Files +## Adding entries -| File | Domain | Populated? | -| ----------------- | ------------------------------------- | ------------- | -| `anatomy.txt` | Latin/Greek anatomical terms | build script | -| `substances.txt` | Exotic active substances (INNs) | TBD | -| `medications.txt` | Exotic medication brand names | TBD | - -## Build scripts - -The `anatomy.txt` is produced by shell pipeline under `../scripts/`: - -``` -./scripts/build-anatomy.sh -``` - -It crawls the German Wikipedia "Kategorie:Anatomie", filters the result -against `hunspell-de-de` to remove everyday German words, and writes the -remainder here. The result is checked into git — we rebuild only when the -source vocabulary materially changes, not on every server build. - -Intermediate files land under `vocab/raw/`, which is gitignored. +Edit `vocabulary.txt` (or drop additional `*.txt` files into this +directory — the loader picks up every `*.txt` non-recursively). Restart +the server afterwards; there is no hot-reload. Keep entries phonetically +distinctive: `Tavor` is a good entry, `Haus` would match innocent words. diff --git a/server/vocab/anatomy.txt b/server/vocab/anatomy.txt deleted file mode 100644 index 0f784ea..0000000 --- a/server/vocab/anatomy.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Anatomical Latin/Greek terms — currently disabled. -# Scope narrowed to medications. If re-enabling, populate via -# ./scripts/build-anatomy.sh and expect eponym-beifang. diff --git a/server/vocab/substances.txt b/server/vocab/substances.txt deleted file mode 100644 index 7e455c7..0000000 --- a/server/vocab/substances.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Exotic active substances (INN names). Population script TBD. -# Candidate source: WHO ATC classification (free, tabular). diff --git a/server/vocab/medications.txt b/server/vocab/vocabulary.txt similarity index 100% rename from server/vocab/medications.txt rename to server/vocab/vocabulary.txt