Feat: Add gazetteer normalization to transcription
This commit integrates the Gazetteer into the transcription pipeline to normalize terminology. The Gazetteer, loaded at startup, is now passed to the transcription worker. Before persisting the transcription text, it is processed through the Gazetteer's `replace` method, ensuring standardized terminology. This normalization is also applied to the generated "oneliner" text. Additionally, the `regenerate_missing_oneliners` function in the recovery module has been updated to accept and utilize the Gazetteer, ensuring that regenerated oneliners also benefit from terminology normalization. A new integration test, `transcribe_worker_normalizes_whisper_output`, has been added to verify that the Whisper output is correctly normalized by the Gazetteer before being saved to the transcript file.
This commit is contained in:
@@ -287,7 +287,8 @@ async fn worker_renames_audio_to_failed_on_whisper_error() {
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let busy = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
transcribe::worker::run(rx, config, client, busy).await;
|
||||
let vocab = std::sync::Arc::new(doctate_server::gazetteer::Gazetteer::empty());
|
||||
transcribe::worker::run(rx, config, client, busy, vocab).await;
|
||||
|
||||
// Audio must have been renamed so recovery skips it next time.
|
||||
assert!(!audio.exists(), "original .m4a still present");
|
||||
@@ -295,6 +296,55 @@ async fn worker_renames_audio_to_failed_on_whisper_error() {
|
||||
assert!(failed.exists(), "expected {} to exist", failed.display());
|
||||
}
|
||||
|
||||
/// The worker must route the Whisper response through the Gazetteer
|
||||
/// before persisting. Mock returns the drift form "Zerebrum"; the
|
||||
/// persisted `.transcript.txt` must contain the canonical "Cerebrum".
|
||||
#[tokio::test]
|
||||
async fn transcribe_worker_normalizes_whisper_output() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/asr"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200).set_body_string("Patient mit Blutung im Zerebrum."),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let case_dir = tmp.path().join("dr_test/ffff");
|
||||
std::fs::create_dir_all(&case_dir).unwrap();
|
||||
let audio = case_dir.join("2026-04-13T10-30-00Z.m4a");
|
||||
std::fs::copy(fixture("sample.m4a"), &audio).unwrap();
|
||||
|
||||
let vocab_dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(vocab_dir.path().join("anatomy.txt"), "Cerebrum\n").unwrap();
|
||||
let vocab = std::sync::Arc::new(
|
||||
doctate_server::gazetteer::Gazetteer::load_dir(vocab_dir.path())
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
assert_eq!(vocab.len(), 1);
|
||||
|
||||
let config = test_config_with_whisper(server.uri());
|
||||
let (tx, rx) = transcribe::channel();
|
||||
tx.send(transcribe::TranscribeJob {
|
||||
audio_path: audio.clone(),
|
||||
user_slug: "dr_test".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
drop(tx);
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let busy = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
transcribe::worker::run(rx, config, client, busy, vocab).await;
|
||||
|
||||
let transcript_path = case_dir.join("2026-04-13T10-30-00Z.transcript.txt");
|
||||
assert!(transcript_path.exists(), "transcript missing");
|
||||
let transcript = std::fs::read_to_string(&transcript_path).unwrap();
|
||||
assert_eq!(transcript, "Patient mit Blutung im Cerebrum.");
|
||||
}
|
||||
|
||||
// -------------------- Ollama client --------------------
|
||||
|
||||
const MODEL: &str = "gemma3:4b";
|
||||
|
||||
Reference in New Issue
Block a user