experiments: pre-LLM gazetteer pass in run_llm_only + new prompt variants

The previous run_llm_only only applied the gazetteer post-LLM, which
silently underestimated the production pipeline: tokens like Inoxaparin
or Klappenvizien are caught pre-LLM in production, but the test tool
left them in the LLM input. Adding `gazetteer.replace()` to each
recording before render_prompt mirrors transcribe/worker.rs:138 exactly,
making sandbox results faithful to production behavior.

New prompt variants kept for the iteration record:
- v3_treue_konsolidiert.txt — the winning variant now in
  server/src/analyze/prompt.rs; redundancies eliminated where they were
  not load-bearing
- v4_treue_kompakt.txt — rejected variant (3/3 unmarked Hypotonie
  hallucinations); kept so future iterations don't repeat the
  experiment

baseline.txt now mirrors the current server prompt (was a stale
pre-v2 snapshot), so future sandbox runs against `baseline.txt`
compare against what is actually shipping.
This commit is contained in:
2026-04-27 23:27:18 +02:00
parent 330e84e473
commit ff53d8bd1b
4 changed files with 72 additions and 14 deletions
+15 -7
View File
@@ -89,9 +89,22 @@ async fn main() -> Result<()> {
}
let llm_system_prompt = load_prompt(&args.llm_prompt)?;
let recordings = load_transcripts(&args.transcripts_dir)?;
let mut recordings = load_transcripts(&args.transcripts_dir)?;
info!(count = recordings.len(), "transcripts loaded");
let gazetteer = Gazetteer::load_dir(&args.vocab_dir)
.await
.with_context(|| format!("loading vocab dir {}", args.vocab_dir.display()))?;
info!(loaded = gazetteer.len(), vocab_dir = %args.vocab_dir.display(), "gazetteer ready");
// Pre-LLM gazetteer pass — mirrors server/src/transcribe/worker.rs:138.
// The tmpdata transcripts were already cleaned with an earlier vocabulary
// snapshot; re-running with the current dir is idempotent for old entries
// and applies any newly added entries (e.g. Enoxaparin, Pumpfunktion).
for r in recordings.iter_mut() {
r.text = gazetteer.replace(&r.text);
}
let analysis_input = AnalysisInput {
last_recording_mtime: recordings
.last()
@@ -105,11 +118,6 @@ async fn main() -> Result<()> {
let llm_variant_id = stem(&args.llm_prompt);
let runs_root = data_runs_root(&case_dir)?;
let gazetteer = Gazetteer::load_dir(&args.vocab_dir)
.await
.with_context(|| format!("loading vocab dir {}", args.vocab_dir.display()))?;
info!(loaded = gazetteer.len(), vocab_dir = %args.vocab_dir.display(), "post-llm gazetteer ready");
for run_index in 1..=args.runs {
let id = run_id("none", &llm_variant_id, run_index);
let run_dir = runs_root.join(&id);
@@ -146,7 +154,7 @@ async fn main() -> Result<()> {
llm_url: settings.llm.url.clone(),
llm_model: settings.llm.model.clone(),
llm_temperature: settings.llm.temperature,
vocab_dir: format!("{} (post-LLM only)", args.vocab_dir.display()),
vocab_dir: format!("{} (pre+post-LLM)", args.vocab_dir.display()),
vocab_loaded: gazetteer.len(),
gazetteer_dict: GazetteerDictMode::NoDict,
timings_ms: timings,