Feat: Add gazetteer normalization to transcription
This commit integrates the Gazetteer into the transcription pipeline to normalize terminology. The Gazetteer, loaded at startup, is now passed to the transcription worker. Before persisting the transcription text, it is processed through the Gazetteer's `replace` method, ensuring standardized terminology. This normalization is also applied to the generated "oneliner" text. Additionally, the `regenerate_missing_oneliners` function in the recovery module has been updated to accept and utilize the Gazetteer, ensuring that regenerated oneliners also benefit from terminology normalization. A new integration test, `transcribe_worker_normalizes_whisper_output`, has been added to verify that the Whisper output is correctly normalized by the Gazetteer before being saved to the transcript file.
This commit is contained in:
+7
-3
@@ -65,8 +65,10 @@ async fn main() {
|
||||
.build()
|
||||
.expect("reqwest client build failed");
|
||||
|
||||
// Gazetteer for pre-LLM proper-name correction. Missing / unreadable
|
||||
// vocab directory is non-fatal — analysis falls back to plain prompts.
|
||||
// Gazetteer: post-AI terminology normalization filter applied to
|
||||
// every AI output (Whisper, oneliner, analyze). Missing / unreadable
|
||||
// vocab directory is non-fatal — the pipeline runs without
|
||||
// normalization.
|
||||
let vocab = Arc::new(match Gazetteer::load_dir(&config.vocab_dir).await {
|
||||
Ok(g) => {
|
||||
info!(
|
||||
@@ -95,17 +97,19 @@ async fn main() {
|
||||
config.clone(),
|
||||
http_client.clone(),
|
||||
transcribe_busy.clone(),
|
||||
vocab.clone(),
|
||||
));
|
||||
{
|
||||
let tx = transcribe_tx.clone();
|
||||
let data_path = config.data_path.clone();
|
||||
let client = http_client.clone();
|
||||
let cfg = config.clone();
|
||||
let vocab = vocab.clone();
|
||||
tokio::spawn(async move {
|
||||
transcribe::recovery::scan_and_enqueue(&data_path, &tx).await;
|
||||
// Then catch up oneliners for cases that crashed between
|
||||
// transcript-write and oneliner-write in a previous run.
|
||||
transcribe::recovery::regenerate_missing_oneliners(&data_path, &client, &cfg).await;
|
||||
transcribe::recovery::regenerate_missing_oneliners(&data_path, &client, &cfg, &vocab).await;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ use tracing::{info, warn};
|
||||
|
||||
use super::{TranscribeJob, TranscribeSender};
|
||||
use crate::config::Config;
|
||||
use crate::gazetteer::Gazetteer;
|
||||
|
||||
/// Walk `data_path/*/` and enqueue every recording pending transcription for
|
||||
/// every user. Intended for startup; the same primitive backs the per-user
|
||||
@@ -144,6 +145,7 @@ pub async fn regenerate_missing_oneliners(
|
||||
data_path: &Path,
|
||||
client: &reqwest::Client,
|
||||
config: &Config,
|
||||
vocab: &Gazetteer,
|
||||
) {
|
||||
let cases = cases_needing_oneliner(data_path).await;
|
||||
if cases.is_empty() {
|
||||
@@ -151,7 +153,7 @@ pub async fn regenerate_missing_oneliners(
|
||||
}
|
||||
info!(count = cases.len(), "Regenerating missing oneliners");
|
||||
for case_dir in cases {
|
||||
super::worker::update_oneliner(&case_dir, client, config).await;
|
||||
super::worker::update_oneliner(&case_dir, client, config, vocab).await;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ use tracing::{error, info, warn};
|
||||
|
||||
use super::{ffmpeg, ollama, whisper, TranscribeReceiver};
|
||||
use crate::config::{Config, WhisperUserSettings};
|
||||
use crate::gazetteer::Gazetteer;
|
||||
use crate::{BusyGuard, WorkerBusy};
|
||||
|
||||
const ONELINER_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
@@ -17,6 +18,7 @@ pub async fn run(
|
||||
config: Arc<Config>,
|
||||
client: reqwest::Client,
|
||||
worker_busy: WorkerBusy,
|
||||
vocab: Arc<Gazetteer>,
|
||||
) {
|
||||
info!("Transcription worker started");
|
||||
let timeout = Duration::from_secs(config.whisper_timeout_seconds);
|
||||
@@ -60,6 +62,11 @@ pub async fn run(
|
||||
}
|
||||
};
|
||||
|
||||
// Normalize terminology against the curated vocab before
|
||||
// persisting. Downstream consumers (oneliner, analyze) read
|
||||
// the canonical path.
|
||||
let text = vocab.replace(&text);
|
||||
|
||||
if let Err(e) = tokio::fs::write(&transcript_path, &text).await {
|
||||
error!(transcript = %transcript_path.display(), error = %e, "writing transcript failed");
|
||||
continue;
|
||||
@@ -76,7 +83,7 @@ pub async fn run(
|
||||
// rule "Spätere Aufnahmen haben Vorrang". Silent / empty cases are
|
||||
// no-ops; LLM errors leave any existing oneliner untouched.
|
||||
if let Some(case_dir) = audio_path.parent() {
|
||||
update_oneliner(case_dir, &client, &config).await;
|
||||
update_oneliner(case_dir, &client, &config, &vocab).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,6 +124,7 @@ pub(crate) async fn update_oneliner(
|
||||
case_dir: &Path,
|
||||
client: &reqwest::Client,
|
||||
config: &Config,
|
||||
vocab: &Gazetteer,
|
||||
) {
|
||||
let path = case_dir.join("oneliner.txt");
|
||||
|
||||
@@ -136,6 +144,7 @@ pub(crate) async fn update_oneliner(
|
||||
.await
|
||||
{
|
||||
Ok(line) => {
|
||||
let line = vocab.replace(&line);
|
||||
if let Err(e) = tokio::fs::write(&path, &line).await {
|
||||
error!(path = %path.display(), error = %e, "writing oneliner failed");
|
||||
} else {
|
||||
|
||||
@@ -287,7 +287,8 @@ async fn worker_renames_audio_to_failed_on_whisper_error() {
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let busy = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
transcribe::worker::run(rx, config, client, busy).await;
|
||||
let vocab = std::sync::Arc::new(doctate_server::gazetteer::Gazetteer::empty());
|
||||
transcribe::worker::run(rx, config, client, busy, vocab).await;
|
||||
|
||||
// Audio must have been renamed so recovery skips it next time.
|
||||
assert!(!audio.exists(), "original .m4a still present");
|
||||
@@ -295,6 +296,55 @@ async fn worker_renames_audio_to_failed_on_whisper_error() {
|
||||
assert!(failed.exists(), "expected {} to exist", failed.display());
|
||||
}
|
||||
|
||||
/// The worker must route the Whisper response through the Gazetteer
|
||||
/// before persisting. Mock returns the drift form "Zerebrum"; the
|
||||
/// persisted `.transcript.txt` must contain the canonical "Cerebrum".
|
||||
#[tokio::test]
|
||||
async fn transcribe_worker_normalizes_whisper_output() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/asr"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200).set_body_string("Patient mit Blutung im Zerebrum."),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let case_dir = tmp.path().join("dr_test/ffff");
|
||||
std::fs::create_dir_all(&case_dir).unwrap();
|
||||
let audio = case_dir.join("2026-04-13T10-30-00Z.m4a");
|
||||
std::fs::copy(fixture("sample.m4a"), &audio).unwrap();
|
||||
|
||||
let vocab_dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(vocab_dir.path().join("anatomy.txt"), "Cerebrum\n").unwrap();
|
||||
let vocab = std::sync::Arc::new(
|
||||
doctate_server::gazetteer::Gazetteer::load_dir(vocab_dir.path())
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
assert_eq!(vocab.len(), 1);
|
||||
|
||||
let config = test_config_with_whisper(server.uri());
|
||||
let (tx, rx) = transcribe::channel();
|
||||
tx.send(transcribe::TranscribeJob {
|
||||
audio_path: audio.clone(),
|
||||
user_slug: "dr_test".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
drop(tx);
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let busy = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
transcribe::worker::run(rx, config, client, busy, vocab).await;
|
||||
|
||||
let transcript_path = case_dir.join("2026-04-13T10-30-00Z.transcript.txt");
|
||||
assert!(transcript_path.exists(), "transcript missing");
|
||||
let transcript = std::fs::read_to_string(&transcript_path).unwrap();
|
||||
assert_eq!(transcript, "Patient mit Blutung im Cerebrum.");
|
||||
}
|
||||
|
||||
// -------------------- Ollama client --------------------
|
||||
|
||||
const MODEL: &str = "gemma3:4b";
|
||||
|
||||
Reference in New Issue
Block a user