Brummel/doctate
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

Remove obsolete vocabulary build scripts

Browse Source 
The build scripts for generating `anatomy.txt` and related files have
been removed as they are no longer used or maintained. The Kölner
Phonetik module was also removed as it was not being utilized.
This commit is contained in:
Brummel
2026-04-16 18:56:08 +02:00
parent 17fb14a741
commit 32f6557d85
10 changed files with 71 additions and 559 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
-1
View File
@@ -4,4 +4,3 @@ users.toml
*.snap.new
repomix-output.xml
tmpdata/
server/vocab/raw/
server/scripts/build-anatomy.sh
-31
View File
@@ -1,31 +0,0 @@
#!/usr/bin/env bash
#
# Build vocab/anatomy.txt from scratch:
# 1. Crawl de.wikipedia.org Kategorie:Anatomie → raw/anatomy-wiki.txt
# 2. Filter against hunspell-de → vocab/anatomy.txt
#
# Idempotent: safe to re-run. Commit vocab/anatomy.txt after a successful
# build so the server uses a deterministic artefact, not a live crawl.
#
# Usage: (from anywhere) bash path/to/build-anatomy.sh
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
VOCAB_DIR="$(cd "$SCRIPT_DIR/../vocab" && pwd)"
RAW_DIR="$VOCAB_DIR/raw"
mkdir -p "$RAW_DIR"
RAW_FILE="$RAW_DIR/anatomy-wiki.txt"
OUT_FILE="$VOCAB_DIR/anatomy.txt"
echo "[1/2] Fetching Wikipedia-de Kategorie:Anatomie..."
"$SCRIPT_DIR/fetch-wiki-anatomy.sh" > "$RAW_FILE"
echo " raw titles: $(wc -l < "$RAW_FILE")"
echo "[2/2] Filtering against hunspell-de..."
"$SCRIPT_DIR/filter-hunspell-de.sh" "$RAW_FILE" "$OUT_FILE"
echo " kept unknown: $(wc -l < "$OUT_FILE")"
echo "Done. Updated: $OUT_FILE"
server/scripts/fetch-wiki-anatomy.sh
-92
View File
@@ -1,92 +0,0 @@
#!/usr/bin/env bash
#
# Crawl the German Wikipedia "Kategorie:Anatomie" (recursively into
# subcategories) and print each page title on stdout, one per line,
# sorted and deduplicated. Progress info goes to stderr.
#
# Dependencies: curl, jq.
# Rate limit: ~10 requests/sec (sleep 0.1 between requests).
# Depth limit: MAX_DEPTH — prevents infinite recursion via cross-links.
#
# Usage: ./fetch-wiki-anatomy.sh > raw/anatomy-wiki.txt
set -euo pipefail
USER_AGENT="doctate-vocab-builder (mould73@gmail.com)"
API="https://de.wikipedia.org/w/api.php"
ROOT_CATEGORY="Kategorie:Anatomie"
MAX_DEPTH=4
for tool in curl jq; do
if ! command -v "$tool" >/dev/null 2>&1; then
echo "Error: $tool is required but not installed." >&2
exit 1
fi
done
PAGES=$(mktemp)
SEEN=$(mktemp)
trap 'rm -f "$PAGES" "$SEEN"' EXIT
# Fetch all pages under one category, recursing into subcategories.
# Args: category_title depth
fetch_category() {
local cat="$1"
local depth="$2"
if [[ $depth -gt $MAX_DEPTH ]]; then
return
fi
if grep -qFx "$cat" "$SEEN"; then
return
fi
echo "$cat" >> "$SEEN"
echo "[depth=$depth] $cat" >&2
local cmcontinue=""
while true; do
local response
if [[ -n "$cmcontinue" ]]; then
response=$(curl -sS -A "$USER_AGENT" -G "$API" \
--data-urlencode "action=query" \
--data-urlencode "list=categorymembers" \
--data-urlencode "cmtitle=$cat" \
--data-urlencode "cmlimit=500" \
--data-urlencode "cmtype=page|subcat" \
--data-urlencode "format=json" \
--data-urlencode "cmcontinue=$cmcontinue")
else
response=$(curl -sS -A "$USER_AGENT" -G "$API" \
--data-urlencode "action=query" \
--data-urlencode "list=categorymembers" \
--data-urlencode "cmtitle=$cat" \
--data-urlencode "cmlimit=500" \
--data-urlencode "cmtype=page|subcat" \
--data-urlencode "format=json")
fi
# Collect page titles (ns=0 is "Main" namespace, i.e. articles).
echo "$response" | jq -r '.query.categorymembers[] | select(.ns == 0) | .title' >> "$PAGES"
# Recurse into subcategories (ns=14).
local subcats
subcats=$(echo "$response" | jq -r '.query.categorymembers[] | select(.ns == 14) | .title')
while IFS= read -r subcat; do
[[ -z "$subcat" ]] && continue
fetch_category "$subcat" $((depth + 1))
done <<< "$subcats"
cmcontinue=$(echo "$response" | jq -r '.continue.cmcontinue // empty')
if [[ -z "$cmcontinue" ]]; then
break
fi
sleep 0.1
done
}
fetch_category "$ROOT_CATEGORY" 0
echo "Collected $(wc -l < "$PAGES") raw titles, emitting sorted-unique..." >&2
sort -u "$PAGES"
server/scripts/filter-hunspell-de.sh
-52
View File
@@ -1,52 +0,0 @@
#!/usr/bin/env bash
#
# Filter a word list to entries NOT recognised by hunspell-de. Uses
# `hunspell -l` (list-unknown-words mode) — a single batch invocation,
# orders of magnitude faster than per-word lookups.
#
# Multi-word input lines are split into words by hunspell; only the
# individual unknown words are kept. That matches our gazetteer's
# token-level matching model (see gazetteer/mod.rs).
#
# Dependencies: hunspell + German dictionary. Package names vary:
# Arch/CachyOS: pacman -S hunspell hunspell-de
# Debian/Ubuntu: apt install hunspell hunspell-de-de
# Fedora: dnf install hunspell hunspell-de
#
# Usage: ./filter-hunspell-de.sh <input> <output>
set -euo pipefail
if [[ $# -ne 2 ]]; then
echo "Usage: $0 <input> <output>" >&2
exit 1
fi
INPUT="$1"
OUTPUT="$2"
# Minimum entry length. Must match MIN_TOKEN_LEN in src/gazetteer/mod.rs.
# 5 is the current value — appropriate for curated medication lists where
# the 4-char eponym-vs-German-word collision class is not present.
MIN_LEN=5
if ! command -v hunspell >/dev/null 2>&1; then
echo "Error: hunspell is not installed. Install it via your package" >&2
echo " manager (e.g. pacman -S hunspell, apt install hunspell)." >&2
exit 1
fi
# Sanity-check: is the de_DE dictionary available?
if ! echo "Haus" | hunspell -d de_DE -a >/dev/null 2>&1; then
echo "Error: hunspell de_DE dictionary not available. Install:" >&2
echo " Arch/CachyOS: pacman -S hunspell-de" >&2
echo " Debian/Ubuntu: apt install hunspell-de-de" >&2
echo " Fedora: dnf install hunspell-de" >&2
exit 1
fi
# `hunspell -l` prints every unknown word from stdin, one per line.
# Drop words shorter than MIN_LEN, then sort -u for dedup + stable order.
hunspell -d de_DE -l < "$INPUT" \
| awk -v min="$MIN_LEN" 'length($0) >= min' \
| sort -u > "$OUTPUT"
server/src/gazetteer/koelner.rs
-238
View File
@@ -1,238 +0,0 @@
//! Kölner Phonetik (Postel 1969) — deterministic sound-hash for German
//! words. Two inputs with the same code are phonetically similar.
//!
//! We use this to find gazetteer entries that *sound like* a
//! misspelled token in the transcript. Example: both `Vomex` (real
//! drug name) and `Womax` (Whisper hallucination) encode to `3648`,
//! so a gazetteer containing `Vomex` can surface it as a candidate
//! correction for `Womax`.
//!
//! Pure function, no I/O, no allocation beyond the returned `String`.
/// Encode `s` as a Kölner-Phonetik code. Output is a sequence of ASCII
/// digits `0`–`8`. Non-alphabetic input chars are silently dropped;
/// case is ignored. Empty input returns an empty string.
///
/// Rules (condensed from Postel 1969 / Wikipedia):
/// - Vowels `A E I J O U Y` → `0`; leading `0` is kept, internal /
/// trailing `0`s are removed in post-processing.
/// - `H` is not encoded at all (neither emitted nor collapsed).
/// - `B` → `1`; `P` → `1`, but `PH` → `3`.
/// - `D T` → `2`, but before `C S Z` → `8`.
/// - `F V W` → `3`.
/// - `G K Q` → `4`.
/// - `C` is context-dependent:
/// * initial + before `A H K L O Q R U X` → `4`
/// * initial otherwise → `8`
/// * after `S Z` → `8`
/// * before `A H K O Q U X` → `4`
/// * otherwise → `8`
/// - `X` → `48`, but after `C K Q` → `8` (those already encoded the 4).
/// - `L` → `5`; `M N` → `6`; `R` → `7`; `S Z` → `8`.
/// - Post: collapse consecutive equal digits, then drop every non-leading `0`.
pub fn encode(s: &str) -> String {
let prepared = prepare(s);
if prepared.is_empty() {
return String::new();
}
let raw = raw_encode(&prepared);
postprocess(&raw)
}
/// Expand umlauts + `ß`, uppercase everything else, drop non-A-Z chars.
/// Returns a `Vec<char>` because subsequent passes need random access
/// and look-ahead, which is awkward on `&str` (UTF-8 byte indices ≠
/// char positions).
fn prepare(s: &str) -> Vec<char> {
let mut out: Vec<char> = Vec::with_capacity(s.len());
for c in s.chars() {
match c {
'ä' | 'Ä' => out.push('A'),
'ö' | 'Ö' => out.push('O'),
'ü' | 'Ü' => out.push('U'),
'ß' => {
out.push('S');
out.push('S');
}
_ => {
for u in c.to_uppercase() {
if u.is_ascii_uppercase() {
out.push(u);
}
}
}
}
}
out
}
/// Per-character encoding with one-char look-ahead and look-behind.
/// Emits ASCII digit chars into the output buffer. `H` emits nothing;
/// `X` may emit two chars.
fn raw_encode(chars: &[char]) -> Vec<char> {
let mut out: Vec<char> = Vec::with_capacity(chars.len() + 2);
for (i, &c) in chars.iter().enumerate() {
let next = chars.get(i + 1).copied();
let prev = if i > 0 { chars.get(i - 1).copied() } else { None };
match c {
'A' | 'E' | 'I' | 'J' | 'O' | 'U' | 'Y' => out.push('0'),
'H' => {} // not encoded
'B' => out.push('1'),
'P' => out.push(if next == Some('H') { '3' } else { '1' }),
'D' | 'T' => out.push(
if matches!(next, Some('C') | Some('S') | Some('Z')) {
'8'
} else {
'2'
},
),
'F' | 'V' | 'W' => out.push('3'),
'G' | 'K' | 'Q' => out.push('4'),
'C' => {
let d = if i == 0 {
// Word-initial C.
if matches!(
next,
Some('A') | Some('H') | Some('K') | Some('L')
| Some('O') | Some('Q') | Some('R')
| Some('U') | Some('X')
) {
'4'
} else {
'8'
}
} else if matches!(prev, Some('S') | Some('Z')) {
// Override: after S/Z always 8, regardless of next.
'8'
} else if matches!(
next,
Some('A') | Some('H') | Some('K') | Some('O')
| Some('Q') | Some('U') | Some('X')
) {
'4'
} else {
'8'
};
out.push(d);
}
'X' => {
if matches!(prev, Some('C') | Some('K') | Some('Q')) {
// Previous char already emitted the '4'; only the '8' is new.
out.push('8');
} else {
out.push('4');
out.push('8');
}
}
'L' => out.push('5'),
'M' | 'N' => out.push('6'),
'R' => out.push('7'),
'S' | 'Z' => out.push('8'),
_ => {} // unreachable after prepare(), but harmless.
}
}
out
}
/// Collapse runs of identical digits into one, then drop every internal
/// or trailing `0`. A leading `0` is preserved because it carries
/// information — the word started with a vowel.
fn postprocess(raw: &[char]) -> String {
let mut dedup: Vec<char> = Vec::with_capacity(raw.len());
for &c in raw {
if dedup.last() != Some(&c) {
dedup.push(c);
}
}
let mut out = String::with_capacity(dedup.len());
for (i, &c) in dedup.iter().enumerate() {
if c == '0' && i > 0 {
continue;
}
out.push(c);
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn encodes_classic_german_surnames() {
// Reference values from the German Wikipedia article on Kölner Phonetik.
assert_eq!(encode("Müller"), "657");
assert_eq!(encode("Meier"), "67");
assert_eq!(encode("Meyer"), "67");
assert_eq!(encode("Mayer"), "67");
assert_eq!(encode("Schmidt"), "862");
assert_eq!(encode("Schmitz"), "868");
}
#[test]
fn treats_umlaut_and_digraph_as_equivalent() {
// Classic test: "ü" and "ue" should normalise to the same code.
assert_eq!(encode("Müller"), encode("Mueller"));
}
/// Regression guard for the ASR-correction use case. Womax (Whisper
/// hallucination) and Vomex (real drug name) must encode identically
/// — that is the mechanism that makes the gazetteer lookup fire.
#[test]
fn vomex_and_womax_collide() {
assert_eq!(encode("Vomex"), encode("Womax"));
assert_eq!(encode("Vomex"), "3648");
}
#[test]
fn empty_and_non_alphabetic_input_yields_empty_code() {
assert_eq!(encode(""), "");
assert_eq!(encode("123 !?"), "");
}
#[test]
fn is_case_insensitive() {
assert_eq!(encode("MÜLLER"), encode("müller"));
assert_eq!(encode("MüLLeR"), encode("Müller"));
}
#[test]
fn leading_vowel_keeps_leading_zero() {
// "Aber" → A=0, B=1, E=0, R=7 → "0107" → dedupe: "0107" → drop non-leading 0: "017".
assert_eq!(encode("Aber"), "017");
}
#[test]
fn c_before_h_at_word_start_codes_4() {
// "Christian": C(initial,next=H→4) H(skip) R=7 I=0 S=8 T(next=I,not CSZ→2) I=0 A=0 N=6
// raw = 4,7,0,8,2,0,0,6 → dedupe = 4,7,0,8,2,0,6 → drop internal 0 = 4,7,8,2,6
assert_eq!(encode("Christian"), "47826");
}
#[test]
fn x_after_c_is_single_digit() {
// "Axel": A=0, X (prev=A, emits 4+8), E=0, L=5 → 0,4,8,0,5 → dedupe same → "0485"
assert_eq!(encode("Axel"), "0485");
// "Knixel": K=4, N=6, I=0, X (prev=I, emits 4+8), E=0, L=5 → 4,6,0,4,8,0,5 → "46485"
assert_eq!(encode("Knixel"), "46485");
}
#[test]
fn ignores_embedded_digits_and_punctuation() {
// "F1-Chip" → prepared: F,C,H,I,P. F=3, C(non-initial,prev=F,next=H→4),
// H(skip), I=0, P(no next=H→1). Raw: 3,4,0,1 → no dedupe → "341".
assert_eq!(encode("F1-Chip"), "341");
}
#[test]
fn ph_at_start_codes_3() {
// "Phil" → P(next=H→3), H(skip), I=0, L=5 → 3,0,5 → "35".
assert_eq!(encode("Phil"), "35");
}
#[test]
fn s_z_before_c_yields_8() {
// "Katze" → K=4, A=0, T(next=Z→8), Z=8, E=0 → 4,0,8,8,0 → dedupe 4,0,8,0 → "48".
assert_eq!(encode("Katze"), "48");
}
}
server/src/gazetteer/mod.rs
+54 -110
View File
@@ -1,32 +1,33 @@
//! Gazetteer: domain-vocabulary lookup that injects correction hints
//! before the analysis LLM runs. See
//! `~/.claude/plans/happy-mapping-snail.md` for the design document.
//! before the analysis LLM runs.
//!
//! The LLM cannot know invented proper names (drug brands, anatomical
//! Latin/Greek terms). Whisper misrenders them phonetically
//! (`Vomex` → `Womax`). This module exposes a static lookup that, given
//! a transcript, finds tokens that are likely typos of a known entry
//! and rewrites them as `token [?Canonical]`. The LLM decides in context
//! The LLM cannot know invented proper names (drug brands, uncommon
//! technical terms). Whisper misrenders them, usually within a couple
//! of edits of the real spelling. This module finds those tokens and
//! rewrites them as `token [?Canonical]`. The LLM decides in context
//! whether to adopt the hint.
//!
//! Matcher: single-stage linear Damerau-Levenshtein scan over all
//! entries, filtered by distance ≤ MAX_EDIT_DISTANCE. At typical sizes
//! (~200 entries × ~50 transcript tokens), the per-pass cost stays well
//! under a millisecond — dwarfed by the downstream LLM call.
pub mod koelner;
use std::collections::{HashMap, HashSet};
use std::collections::HashSet;
use std::io;
use std::path::Path;
use strsim::damerau_levenshtein;
/// Minimum char count for *both* gazetteer entries and transcript tokens.
/// Calibrated against curated medication lists: 5 keeps prototypical
/// 5-char brand names (Vomex, Tavor, Lasix, Nasic) while still blocking
/// the 4-char deutsche Alltagswort ↔ eponym collisions that unkurated
/// crawls produce (Bein↔Behn, nach↔Nuck).
/// Calibrated against curated vocabulary: 5 keeps prototypical 5-char
/// brand names (Vomex, Tavor, Lasix) while still blocking 4-char
/// collisions between everyday German words and short proper nouns
/// (Bein↔Behn, nach↔Nuck).
const MIN_TOKEN_LEN: usize = 5;
/// Maximum Damerau-Levenshtein distance for a candidate to qualify as
/// a typo of a gazetteer entry. Distance 0 is excluded — the `exact`
/// set short-circuits that case.
/// Maximum Damerau-Levenshtein distance for a candidate to qualify as a
/// typo of a gazetteer entry. Distance 0 is handled by the `exact`
/// short-circuit; distances > 2 are rejected as too speculative.
const MAX_EDIT_DISTANCE: usize = 2;
/// Domain-specific vocabulary, populated once at server start from
@@ -34,23 +35,22 @@ const MAX_EDIT_DISTANCE: usize = 2;
/// via `Arc<Gazetteer>`.
#[derive(Debug, Default)]
pub struct Gazetteer {
/// Kölner-Phonetik code → canonical entries sharing that code. The
/// canonicals keep their original casing for display.
by_phonetic: HashMap<String, Vec<Box<str>>>,
/// Canonical entries in original casing — source of truth for both
/// scanning and emission.
entries: Vec<Box<str>>,
/// Lowercased canonicals; short-circuits annotation when the token
/// is already spelled correctly.
/// already matches a known entry exactly.
exact: HashSet<Box<str>>,
}
impl Gazetteer {
/// Empty gazetteer. `annotate` is a pure pass-through on this instance.
pub fn empty() -> Self {
Self::default()
}
/// Load every `*.txt` file (non-recursive) under `dir`. Blank lines
/// and lines beginning with `#` are skipped. Case-insensitive
/// duplicates are silently collapsed.
/// duplicates across files are silently collapsed.
pub async fn load_dir(dir: &Path) -> io::Result<Self> {
let mut g = Self::default();
let mut entries = tokio::fs::read_dir(dir).await?;
@@ -71,40 +71,29 @@ impl Gazetteer {
Ok(g)
}
/// Number of unique entries (case-insensitive).
pub fn len(&self) -> usize {
self.exact.len()
self.entries.len()
}
pub fn is_empty(&self) -> bool {
self.exact.is_empty()
self.entries.is_empty()
}
fn insert(&mut self, entry: &str) {
// Skip short entries — they cause noisy phonetic collisions with
// ordinary German words. Same threshold as token matching.
if entry.chars().count() < MIN_TOKEN_LEN {
return;
}
let lower = entry.to_lowercase();
if !self.exact.insert(lower.into_boxed_str()) {
// Duplicate (case-insensitive) — don't add to phonetic index either.
// Duplicate (case-insensitive) — already indexed.
return;
}
let code = koelner::encode(entry);
if code.is_empty() {
return;
}
self.by_phonetic
.entry(code)
.or_default()
.push(entry.into());
self.entries.push(entry.into());
}
/// Annotate `text` by appending ` [?Canonical]` after any word-token
/// that looks like a misspelling of a gazetteer entry. Non-matching
/// tokens and all non-word segments (whitespace, punctuation, digits)
/// pass through byte-identical.
/// that's within edit-distance 2 of a gazetteer entry (and not an
/// exact match). Non-word segments pass through byte-identical.
pub fn annotate(&self, text: &str) -> String {
if self.is_empty() {
return text.to_string();
@@ -136,45 +125,15 @@ impl Gazetteer {
return None;
}
// Stage 1: phonetic-indexed lookup. Fast, filters out the vast
// majority of non-matches in O(1). Catches same-code neighbours
// like Klexane↔Clexane, Diacepam↔Diazepam.
let code = koelner::encode(token);
if !code.is_empty()
&& let Some(candidates) = self.by_phonetic.get(&code)
&& let Some(hit) = best_candidate(&lower, candidates.iter().map(|c| c.as_ref()))
{
return Some(hit);
}
// Stage 2: brute-force edit-distance over every entry. Catches
// ASR errors where Whisper substitutes phonetically-distinct
// consonants (Carvenilol↔Carvedilol: N↔D) or drops a letter
// that shifts the Kölner code (Acoxia↔Arcoxia: missing R).
// O(n) per unknown token; at n≈200 entries and ~50 tokens per
// transcript the absolute cost stays well under a millisecond.
best_candidate(
&lower,
self.by_phonetic.values().flatten().map(|c| c.as_ref()),
)
self.entries
.iter()
.map(|c| (damerau_levenshtein(&lower, &c.to_lowercase()), c.as_ref()))
.filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE)
.min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(b.1)))
.map(|(_, c)| c)
}
}
/// Select the canonical entry from `candidates` that is closest to
/// `lower_token` by Damerau-Levenshtein distance, within the allowed
/// threshold. Distance 0 is rejected because that's already handled by
/// the `exact` short-circuit. Ties break alphabetically for determinism.
fn best_candidate<'a>(
lower_token: &str,
candidates: impl Iterator<Item = &'a str>,
) -> Option<&'a str> {
candidates
.map(|c| (damerau_levenshtein(lower_token, &c.to_lowercase()), c))
.filter(|(d, _)| *d >= 1 && *d <= MAX_EDIT_DISTANCE)
.min_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(b.1)))
.map(|(_, c)| c)
}
/// A slice of the input text — either a word or a non-word run.
enum Segment<'a> {
Word(&'a str),
@@ -212,8 +171,6 @@ mod tests {
use super::*;
use tempfile::tempdir;
/// Build a gazetteer from an in-memory list of canonical entries —
/// avoids filesystem I/O in the matcher tests.
fn from_entries(entries: &[&str]) -> Gazetteer {
let mut g = Gazetteer::empty();
for e in entries {
@@ -229,12 +186,8 @@ mod tests {
assert_eq!(g.annotate(input), input);
}
/// Core regression: a token that's a phonetic neighbor at edit
/// distance ≤ 2 gets a hint appended. Cerebrum and Zerebrum both
/// encode to Kölner `87176` and differ by one char — a plausible
/// Whisper-style transcription slip.
#[test]
fn annotates_typo_for_phonetic_neighbor() {
fn annotates_typo_within_distance() {
let g = from_entries(&["Cerebrum"]);
let out = g.annotate("Patient mit Blutung im Zerebrum.");
assert!(
@@ -248,18 +201,18 @@ mod tests {
let g = from_entries(&["Cerebrum"]);
assert_eq!(
g.annotate("Patient mit Blutung im Cerebrum."),
"Patient mit Blutung im Cerebrum.",
"Patient mit Blutung im Cerebrum."
);
}
/// Lowercase variant of an entry is still "exact" — we never nag
/// the LLM with a hint for a token that's only case-different.
/// Lowercase variant of an entry is still "exact" — we never nag the
/// LLM with a hint for a token that differs only in case.
#[test]
fn noop_on_case_only_difference() {
let g = from_entries(&["Cerebrum"]);
assert_eq!(
g.annotate("Patient mit Blutung im cerebrum."),
"Patient mit Blutung im cerebrum.",
"Patient mit Blutung im cerebrum."
);
}
@@ -270,51 +223,42 @@ mod tests {
assert_eq!(g.annotate(input), input);
}
/// Gazetteer entries shorter than MIN_TOKEN_LEN are rejected at load
/// time. This prevents 4-char eponyms (Nuck, Behn, Bell, …) from
/// generating noisy false-positive matches against everyday German
/// words — the empirical reason MIN_TOKEN_LEN exists.
/// Entries shorter than MIN_TOKEN_LEN are rejected at load time —
/// prevents short-proper-noun collisions with everyday German.
#[test]
fn rejects_gazetteer_entries_below_min_length() {
fn rejects_entries_below_min_length() {
let g = from_entries(&["Nuck", "Bein", "Behn"]);
assert_eq!(g.len(), 0, "short entries must be dropped");
// Input with a matching short word passes through untouched.
let input = "Patient klagt nach schwerem Heben.";
assert_eq!(g.annotate(input), input);
}
/// Edit-distance fallback: Whisper substitutes phonetically-distinct
/// consonants (here N↔D in Carvenilol↔Carvedilol). The Kölner codes
/// differ (N=6 vs D=2), so stage-1 misses — the brute-force stage
/// must catch the distance-1 match.
/// Classic ASR-substitution case (N↔D) — distance 1 match.
#[test]
fn edit_distance_fallback_catches_consonant_substitution() {
fn annotates_consonant_substitution() {
let g = from_entries(&["Carvedilol"]);
let out = g.annotate("Patient nimmt Carvenilol.");
assert!(
out.contains("Carvenilol [?Carvedilol]"),
"expected fallback annotation, got: {out:?}"
"got: {out:?}"
);
}
/// Edit-distance fallback: Whisper drops a consonant (leading B in
/// Berodual). Kölner codes are completely different — only the
/// linear edit-distance scan saves the match.
/// Dropped leading consonant — edit-distance 1 from Berodual.
#[test]
fn edit_distance_fallback_catches_dropped_consonant() {
fn annotates_dropped_consonant() {
let g = from_entries(&["Berodual"]);
let out = g.annotate("Patient bekommt Erodual.");
assert!(
out.contains("Erodual [?Berodual]"),
"expected fallback annotation, got: {out:?}"
"got: {out:?}"
);
}
/// With two phonetically-colliding candidates, the one with smaller
/// Damerau-Levenshtein distance wins. Zerebrum↔Cerebrum = 1,
/// Zerebrum↔Cerebrom = 2 → Cerebrum is chosen.
/// With multiple in-range candidates, the closest by
/// Damerau-Levenshtein wins; ties break alphabetically.
#[test]
fn picks_closest_candidate_by_edit_distance() {
fn picks_closest_candidate() {
let g = from_entries(&["Cerebrum", "Cerebrom"]);
let out = g.annotate("Patient mit Blutung im Zerebrum.");
assert!(
@@ -336,7 +280,7 @@ mod tests {
tokio::fs::write(dir.path().join("meds.txt"), "Aspirin\nIbuprofen\n")
.await
.unwrap();
tokio::fs::write(dir.path().join("anatomy.txt"), "Coracoideus\n")
tokio::fs::write(dir.path().join("more.txt"), "Coracoideus\n")
.await
.unwrap();
let g = Gazetteer::load_dir(dir.path()).await.unwrap();
@@ -348,7 +292,7 @@ mod tests {
let dir = tempdir().unwrap();
tokio::fs::write(
dir.path().join("v.txt"),
"# medications\nAspirin\n\n \n# another\nIbuprofen\n",
"# meds\nAspirin\n\n \n# another\nIbuprofen\n",
)
.await
.unwrap();
server/vocab/README.md
+17 -30
View File
@@ -1,35 +1,22 @@
# Gazetteer vocabulary files
# Gazetteer vocabulary
These `*.txt` files are loaded once at server startup and used to annotate
Whisper transcripts with correction hints of the form `Token [?Canonical]`
before the analysis LLM runs. Design: `~/.claude/plans/happy-mapping-snail.md`.
Plain list of domain-specific words the analysis LLM might not know —
medication brand names, proper nouns, colleagues, technical terms. Loaded
once at server start. Used to annotate close edit-distance neighbours in
Whisper transcripts as `Token [?Canonical]` for LLM-side review.
## File format
## Format
- One entry per line, UTF-8, no surrounding whitespace required.
- Lines that are blank or start with `#` are skipped.
- Case-insensitive. Duplicates across files are collapsed.
- No header row, no columns — this is a plain word list.
- One entry per line, UTF-8.
- Blank lines and lines starting with `#` are skipped.
- Matching is case-insensitive; original casing is preserved for display.
- Entries must be ≥5 characters (shorter tokens collide too often with
everyday German words).
- Duplicates (case-insensitive, across files) are collapsed automatically.
## Files
## Adding entries
| File | Domain | Populated? |
| ----------------- | ------------------------------------- | ------------- |
| `anatomy.txt` | Latin/Greek anatomical terms | build script |
| `substances.txt` | Exotic active substances (INNs) | TBD |
| `medications.txt` | Exotic medication brand names | TBD |
## Build scripts
The `anatomy.txt` is produced by shell pipeline under `../scripts/`:
```
./scripts/build-anatomy.sh
```
It crawls the German Wikipedia "Kategorie:Anatomie", filters the result
against `hunspell-de-de` to remove everyday German words, and writes the
remainder here. The result is checked into git — we rebuild only when the
source vocabulary materially changes, not on every server build.
Intermediate files land under `vocab/raw/`, which is gitignored.
Edit `vocabulary.txt` (or drop additional `*.txt` files into this
directory — the loader picks up every `*.txt` non-recursively). Restart
the server afterwards; there is no hot-reload. Keep entries phonetically
distinctive: `Tavor` is a good entry, `Haus` would match innocent words.
server/vocab/anatomy.txt
-3
View File
@@ -1,3 +0,0 @@
# Anatomical Latin/Greek terms — currently disabled.
# Scope narrowed to medications. If re-enabling, populate via
# ./scripts/build-anatomy.sh and expect eponym-beifang.
server/vocab/substances.txt
-2
View File
@@ -1,2 +0,0 @@
# Exotic active substances (INN names). Population script TBD.
# Candidate source: WHO ATC classification (free, tabular).
server/vocab/medications.txt → server/vocab/vocabulary.txt
View File