Feat: Add gazetteer normalization to transcription
This commit integrates the Gazetteer into the transcription pipeline to normalize terminology. The Gazetteer, loaded at startup, is now passed to the transcription worker. Before persisting the transcription text, it is processed through the Gazetteer's `replace` method, ensuring standardized terminology. This normalization is also applied to the generated "oneliner" text. Additionally, the `regenerate_missing_oneliners` function in the recovery module has been updated to accept and utilize the Gazetteer, ensuring that regenerated oneliners also benefit from terminology normalization. A new integration test, `transcribe_worker_normalizes_whisper_output`, has been added to verify that the Whisper output is correctly normalized by the Gazetteer before being saved to the transcript file.
This commit is contained in:
@@ -4,6 +4,7 @@ use tracing::{info, warn};
|
||||
|
||||
use super::{TranscribeJob, TranscribeSender};
|
||||
use crate::config::Config;
|
||||
use crate::gazetteer::Gazetteer;
|
||||
|
||||
/// Walk `data_path/*/` and enqueue every recording pending transcription for
|
||||
/// every user. Intended for startup; the same primitive backs the per-user
|
||||
@@ -144,6 +145,7 @@ pub async fn regenerate_missing_oneliners(
|
||||
data_path: &Path,
|
||||
client: &reqwest::Client,
|
||||
config: &Config,
|
||||
vocab: &Gazetteer,
|
||||
) {
|
||||
let cases = cases_needing_oneliner(data_path).await;
|
||||
if cases.is_empty() {
|
||||
@@ -151,7 +153,7 @@ pub async fn regenerate_missing_oneliners(
|
||||
}
|
||||
info!(count = cases.len(), "Regenerating missing oneliners");
|
||||
for case_dir in cases {
|
||||
super::worker::update_oneliner(&case_dir, client, config).await;
|
||||
super::worker::update_oneliner(&case_dir, client, config, vocab).await;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ use tracing::{error, info, warn};
|
||||
|
||||
use super::{ffmpeg, ollama, whisper, TranscribeReceiver};
|
||||
use crate::config::{Config, WhisperUserSettings};
|
||||
use crate::gazetteer::Gazetteer;
|
||||
use crate::{BusyGuard, WorkerBusy};
|
||||
|
||||
const ONELINER_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
@@ -17,6 +18,7 @@ pub async fn run(
|
||||
config: Arc<Config>,
|
||||
client: reqwest::Client,
|
||||
worker_busy: WorkerBusy,
|
||||
vocab: Arc<Gazetteer>,
|
||||
) {
|
||||
info!("Transcription worker started");
|
||||
let timeout = Duration::from_secs(config.whisper_timeout_seconds);
|
||||
@@ -60,6 +62,11 @@ pub async fn run(
|
||||
}
|
||||
};
|
||||
|
||||
// Normalize terminology against the curated vocab before
|
||||
// persisting. Downstream consumers (oneliner, analyze) read
|
||||
// the canonical path.
|
||||
let text = vocab.replace(&text);
|
||||
|
||||
if let Err(e) = tokio::fs::write(&transcript_path, &text).await {
|
||||
error!(transcript = %transcript_path.display(), error = %e, "writing transcript failed");
|
||||
continue;
|
||||
@@ -76,7 +83,7 @@ pub async fn run(
|
||||
// rule "Spätere Aufnahmen haben Vorrang". Silent / empty cases are
|
||||
// no-ops; LLM errors leave any existing oneliner untouched.
|
||||
if let Some(case_dir) = audio_path.parent() {
|
||||
update_oneliner(case_dir, &client, &config).await;
|
||||
update_oneliner(case_dir, &client, &config, &vocab).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,6 +124,7 @@ pub(crate) async fn update_oneliner(
|
||||
case_dir: &Path,
|
||||
client: &reqwest::Client,
|
||||
config: &Config,
|
||||
vocab: &Gazetteer,
|
||||
) {
|
||||
let path = case_dir.join("oneliner.txt");
|
||||
|
||||
@@ -136,6 +144,7 @@ pub(crate) async fn update_oneliner(
|
||||
.await
|
||||
{
|
||||
Ok(line) => {
|
||||
let line = vocab.replace(&line);
|
||||
if let Err(e) = tokio::fs::write(&path, &line).await {
|
||||
error!(path = %path.display(), error = %e, "writing oneliner failed");
|
||||
} else {
|
||||
|
||||
Reference in New Issue
Block a user