experiments: pre-LLM gazetteer pass in run_llm_only + new prompt variants
The previous run_llm_only only applied the gazetteer post-LLM, which silently underestimated the production pipeline: tokens like Inoxaparin or Klappenvizien are caught pre-LLM in production, but the test tool left them in the LLM input. Adding `gazetteer.replace()` to each recording before render_prompt mirrors transcribe/worker.rs:138 exactly, making sandbox results faithful to production behavior. New prompt variants kept for the iteration record: - v3_treue_konsolidiert.txt — the winning variant now in server/src/analyze/prompt.rs; redundancies eliminated where they were not load-bearing - v4_treue_kompakt.txt — rejected variant (3/3 unmarked Hypotonie hallucinations); kept so future iterations don't repeat the experiment baseline.txt now mirrors the current server prompt (was a stale pre-v2 snapshot), so future sandbox runs against `baseline.txt` compare against what is actually shipping.
This commit is contained in:
@@ -89,9 +89,22 @@ async fn main() -> Result<()> {
|
||||
}
|
||||
|
||||
let llm_system_prompt = load_prompt(&args.llm_prompt)?;
|
||||
let recordings = load_transcripts(&args.transcripts_dir)?;
|
||||
let mut recordings = load_transcripts(&args.transcripts_dir)?;
|
||||
info!(count = recordings.len(), "transcripts loaded");
|
||||
|
||||
let gazetteer = Gazetteer::load_dir(&args.vocab_dir)
|
||||
.await
|
||||
.with_context(|| format!("loading vocab dir {}", args.vocab_dir.display()))?;
|
||||
info!(loaded = gazetteer.len(), vocab_dir = %args.vocab_dir.display(), "gazetteer ready");
|
||||
|
||||
// Pre-LLM gazetteer pass — mirrors server/src/transcribe/worker.rs:138.
|
||||
// The tmpdata transcripts were already cleaned with an earlier vocabulary
|
||||
// snapshot; re-running with the current dir is idempotent for old entries
|
||||
// and applies any newly added entries (e.g. Enoxaparin, Pumpfunktion).
|
||||
for r in recordings.iter_mut() {
|
||||
r.text = gazetteer.replace(&r.text);
|
||||
}
|
||||
|
||||
let analysis_input = AnalysisInput {
|
||||
last_recording_mtime: recordings
|
||||
.last()
|
||||
@@ -105,11 +118,6 @@ async fn main() -> Result<()> {
|
||||
let llm_variant_id = stem(&args.llm_prompt);
|
||||
let runs_root = data_runs_root(&case_dir)?;
|
||||
|
||||
let gazetteer = Gazetteer::load_dir(&args.vocab_dir)
|
||||
.await
|
||||
.with_context(|| format!("loading vocab dir {}", args.vocab_dir.display()))?;
|
||||
info!(loaded = gazetteer.len(), vocab_dir = %args.vocab_dir.display(), "post-llm gazetteer ready");
|
||||
|
||||
for run_index in 1..=args.runs {
|
||||
let id = run_id("none", &llm_variant_id, run_index);
|
||||
let run_dir = runs_root.join(&id);
|
||||
@@ -146,7 +154,7 @@ async fn main() -> Result<()> {
|
||||
llm_url: settings.llm.url.clone(),
|
||||
llm_model: settings.llm.model.clone(),
|
||||
llm_temperature: settings.llm.temperature,
|
||||
vocab_dir: format!("{} (post-LLM only)", args.vocab_dir.display()),
|
||||
vocab_dir: format!("{} (pre+post-LLM)", args.vocab_dir.display()),
|
||||
vocab_loaded: gazetteer.len(),
|
||||
gazetteer_dict: GazetteerDictMode::NoDict,
|
||||
timings_ms: timings,
|
||||
|
||||
Reference in New Issue
Block a user