Update hunspell filter to use min token length
The `filter-hunspell-de.sh` script now uses a `MIN_LEN` variable that is synchronized with the `MIN_TOKEN_LEN` constant in `src/gazetteer/mod.rs`. This ensures that short tokens, which are prone to false positives due to collisions with common German words, are filtered out by both the gazetteer loading and the Hunspell dictionary processing. Additionally, the script's dependency instructions have been updated to be more comprehensive, listing package manager commands for Arch/CachyOS, Debian/Ubuntu, and Fedora. The `awk` command has been introduced to filter out tokens shorter than `MIN_LEN` before sorting and deduplicating. The `Gazetteer::insert` method has been updated to skip entries shorter than `MIN_TOKEN_LEN` to prevent noisy phonetic collisions. The `best_candidate` helper function has been extracted to improve readability and structure. The test cases have been updated to reflect the change in `MIN_TOKEN_LEN` from 4 to 5 and to use more relevant examples for the updated functionality, such as "Cerebrum" and "Zerebrum". The `anatomy.txt` file has been updated to reflect that anatomical terms are currently disabled. The `medications.txt` file has been significantly expanded with a curated list of German medication brand and active ingredient names.
This commit is contained in:
@@ -360,6 +360,131 @@ async fn analyze_worker_writes_document_via_wiremock() {
|
||||
let _ = tokio::time::timeout(Duration::from_secs(2), handle).await;
|
||||
}
|
||||
|
||||
/// The analyze worker must route transcripts through the Gazetteer before
|
||||
/// sending them to the LLM. "Zerebrum" shares a Kölner-Phonetik code with
|
||||
/// the gazetteer entry "Cerebrum" (both encode to `87176`) and differs by
|
||||
/// exactly one character — so the annotation `Zerebrum [?Cerebrum]` must
|
||||
/// end up in the LLM request body.
|
||||
#[tokio::test]
|
||||
async fn analyze_worker_annotates_with_gazetteer() {
|
||||
let tmp = unique_tmp("g");
|
||||
let case_dir = tmp.join("dr_a/22222222-2222-2222-2222-222222222222");
|
||||
std::fs::create_dir_all(&case_dir).unwrap();
|
||||
|
||||
let input = json!({
|
||||
"last_recording_mtime": "2026-04-15T10:00:00Z",
|
||||
"recordings": [
|
||||
{ "recorded_at": "2026-04-15T10:00:00Z", "text": "Patient mit Blutung im Zerebrum." }
|
||||
]
|
||||
});
|
||||
std::fs::write(
|
||||
case_dir.join("analysis_input.json"),
|
||||
serde_json::to_vec_pretty(&input).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Vocab directory with a single anatomy entry.
|
||||
let vocab_dir = unique_tmp("vg");
|
||||
std::fs::create_dir_all(&vocab_dir).unwrap();
|
||||
std::fs::write(vocab_dir.join("anatomy.txt"), "Cerebrum\n").unwrap();
|
||||
let vocab = Arc::new(
|
||||
doctate_server::gazetteer::Gazetteer::load_dir(&vocab_dir)
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
assert_eq!(vocab.len(), 1);
|
||||
|
||||
// Mock LLM: responds 200 unconditionally. The assertion is on the
|
||||
// *request body*, not the response.
|
||||
let mock = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"choices": [{
|
||||
"message": { "content": "Patient mit Blutung im ==Cerebrum==." }
|
||||
}]
|
||||
})))
|
||||
.mount(&mock)
|
||||
.await;
|
||||
|
||||
let config = config_with_llm(tmp.clone(), mock.uri());
|
||||
let (tx, rx) = analyze::channel();
|
||||
let client = reqwest::Client::new();
|
||||
let busy = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let handle = tokio::spawn(analyze::worker::run(rx, config, client, busy, vocab));
|
||||
|
||||
tx.send(analyze::AnalyzeJob {
|
||||
case_dir: case_dir.clone(),
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let document_path = case_dir.join("document.md");
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(5);
|
||||
while !document_path.exists() {
|
||||
if std::time::Instant::now() > deadline {
|
||||
panic!("document.md not written within 5s");
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
|
||||
// Inspect what actually went to the LLM.
|
||||
let requests = mock.received_requests().await.unwrap();
|
||||
assert_eq!(requests.len(), 1, "expected exactly one LLM request");
|
||||
let body = String::from_utf8(requests[0].body.clone()).unwrap();
|
||||
assert!(
|
||||
body.contains("Zerebrum [?Cerebrum]"),
|
||||
"expected gazetteer annotation in LLM request body, got:\n{body}",
|
||||
);
|
||||
|
||||
drop(tx);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(2), handle).await;
|
||||
}
|
||||
|
||||
/// Manual debug utility — not part of the default test set.
|
||||
/// Usage:
|
||||
/// DRY_CASE_DIR=/path/to/case cargo test --test analyze_test annotate_real_case_dry -- --ignored --nocapture
|
||||
///
|
||||
/// Prints each transcript in a given case dir side-by-side with its
|
||||
/// gazetteer-annotated version. Useful to eyeball false-positives /
|
||||
/// false-negatives on real cases.
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn annotate_real_case_dry() {
|
||||
let vocab_dir = std::env::var("DRY_VOCAB_DIR")
|
||||
.unwrap_or_else(|_| "/home/brummel/dev/doctate/server/vocab".into());
|
||||
let case_dir = std::env::var("DRY_CASE_DIR").unwrap_or_else(|_| {
|
||||
"/home/brummel/dev/doctate/tmpdata/dr_mueller/e817d8b8-43eb-436a-8364-76b80a1e627b".into()
|
||||
});
|
||||
|
||||
let vocab = doctate_server::gazetteer::Gazetteer::load_dir(Path::new(&vocab_dir))
|
||||
.await
|
||||
.expect("load vocab");
|
||||
println!("Gazetteer entries: {}", vocab.len());
|
||||
|
||||
let mut names: Vec<_> = std::fs::read_dir(&case_dir)
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok().map(|e| e.path()))
|
||||
.filter(|p| {
|
||||
p.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.map(|s| s.ends_with(".transcript.txt"))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.collect();
|
||||
names.sort();
|
||||
|
||||
for path in names {
|
||||
let text = std::fs::read_to_string(&path).unwrap();
|
||||
let annotated = vocab.annotate(&text);
|
||||
println!("\n=== {} ===", path.file_name().unwrap().to_string_lossy());
|
||||
println!("--- original ---\n{text}");
|
||||
println!("--- annotated ---\n{annotated}");
|
||||
if annotated == text {
|
||||
println!("[no annotations applied]");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Recovery
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user