fix: separate transient from permanent transcribe failures

Before, every Whisper error (5xx, timeout, Minerva down, corrupt audio,
4xx) renamed `<ts>.m4a` to `<ts>.m4a.failed` uniformly, turning transient
outages into permanent sackgassen that only a manual admin reset could
undo. Cases whose every recording was `.m4a.failed` stuck without a
persisted oneliner state; the UI masked them via a fallback in
`compute_oneliner_display`.

The core insight: a recoverable error is not an error. Transient Whisper
failures now leave the audio as plain `.m4a` so the existing page-load
heal (`enqueue_pending_for_user`) re-enqueues it on the next refresh —
which is exactly what happens when Minerva comes back. No new sidecar,
no retry count, no scheduler.

Changes:
- `WhisperError::is_transient` classifies `Http`, 5xx, 408, 429 as
  transient; `Io` and other 4xx as permanent.
- Transcribe worker: transient → info log + continue (audio stays .m4a);
  permanent → `mark_failed` as before.
- `has_any_transcript` counts `.m4a.failed` as terminal, so
  `update_oneliner` runs for failed-only cases and settles
  `OnelinerState::Empty` (analogous to the silent-only fix in 4531f85).
- New verdrängender `fehler`-Badge (#c00) in case list and detail when
  at least one `.m4a.failed` exists; recording-level message shortened
  to plain "Transkription fehlgeschlagen".

Tests:
- New integration: transient 503 leaves `.m4a` intact, heal recovers.
- New integration: `.m4a.failed`-only case settles to
  `OnelinerState::Empty` without calling Ollama.
- New unit: `is_transient` table test across relevant status codes.
- New unit: `has_any_transcript` returns true for `.m4a.failed`-only
  case and false for pending-only case.
- Existing worker test retargeted from 500 to 400 and renamed; added
  companion `worker_leaves_m4a_intact_on_transient_whisper_error`.
This commit is contained in:
2026-04-21 14:39:31 +02:00
parent 4531f85b13
commit 1f32d4dd23
10 changed files with 502 additions and 18 deletions
@@ -0,0 +1,99 @@
//! Regression test: a case whose recordings are all `.m4a.failed`
//! (permanent transcription failures — ffmpeg corrupt audio or Whisper
//! 4xx) must produce an `OnelinerState::Empty` automatically, so the UI
//! has a durable terminal state instead of relying on the on-render
//! `compute_oneliner_display` fallback.
//!
//! The historical bug path: `.m4a.failed`-only cases were skipped by the
//! recovery scan (`has_any_transcript` looked for `*.transcript.txt`
//! only), so `update_oneliner` never ran, so no `oneliner.json` was
//! written. The UI masked the symptom with its `non_failed_count == 0`
//! fallback — but a future refactor of that fallback would regress the
//! case. The fix makes `has_any_transcript` count `.m4a.failed` too, so
//! `update_oneliner` runs, sees no content transcripts, and settles the
//! case to `Empty` through the same terminal branch used for silent-only
//! cases.
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use doctate_common::oneliners::OnelinerState;
use doctate_server::config::Config;
use doctate_server::events;
use doctate_server::gazetteer::Gazetteer;
use doctate_server::{
AnalyzeBusy, OnelinerHealBusy, PipelineState, TranscribeBusy, WorkerBusy, analyze, transcribe,
};
use tempfile::tempdir;
use wiremock::matchers::{method, path as wm_path};
use wiremock::{Mock, MockServer, ResponseTemplate};
#[tokio::test]
async fn failed_only_case_settles_to_empty_without_llm_call() {
let data = tempdir().unwrap();
let slug = "dr_test";
let user_root = data.path().join(slug);
let case_dir = user_root.join("case-failed");
std::fs::create_dir_all(&case_dir).unwrap();
// Seed: only a `.m4a.failed` file. No transcript, no plain .m4a.
std::fs::write(case_dir.join("2026-04-15T20-30-00Z.m4a.failed"), b"x").unwrap();
// Ollama must NOT be called — nothing to summarize for a failed-only
// case. `.expect(0)` fails the test (on MockServer drop) if any
// request arrived.
let mock = MockServer::start().await;
Mock::given(method("POST"))
.and(wm_path("/api/chat"))
.respond_with(ResponseTemplate::new(200))
.expect(0)
.mount(&mock)
.await;
let mut cfg = Config::test_default();
cfg.data_path = data.path().to_path_buf();
cfg.ollama_url = mock.uri();
let config = Arc::new(cfg);
let vocab = Arc::new(Gazetteer::empty());
let events_tx = events::channel();
let http_client = reqwest::Client::new();
let (tx_a, _rx_a) = analyze::channel();
let (tx_t, _rx_t) = transcribe::channel();
let heal_busy: WorkerBusy = Arc::new(AtomicBool::new(false));
let pipeline = PipelineState {
analyze_busy: AnalyzeBusy(Arc::new(AtomicBool::new(false))),
analyze_tx: tx_a,
transcribe_busy: TranscribeBusy(Arc::new(AtomicBool::new(false))),
transcribe_tx: tx_t,
oneliner_heal_busy: OnelinerHealBusy(heal_busy.clone()),
};
pipeline
.heal_orphans_if_idle(&user_root, slug, &http_client, &config, &vocab, &events_tx)
.await;
// Heal spawn drops the busy flag on completion — poll until idle.
let start = std::time::Instant::now();
while heal_busy.load(Ordering::Acquire) {
if start.elapsed() > Duration::from_secs(5) {
panic!("oneliner heal did not finish within 5s");
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
let path = case_dir.join("oneliner.json");
assert!(
path.exists(),
"expected oneliner.json to be written for failed-only case"
);
let bytes = std::fs::read(&path).unwrap();
let state: OnelinerState = serde_json::from_slice(&bytes).unwrap();
assert!(
matches!(state, OnelinerState::Empty { .. }),
"expected OnelinerState::Empty, got {state:?}"
);
// Mock.drop() runs here — `.expect(0)` panics if Ollama was touched.
}
+62 -2
View File
@@ -276,12 +276,17 @@ fn test_config_with_whisper(whisper_url: String) -> Arc<Config> {
})
}
/// Permanent Whisper errors (4xx other than 408/429) must rename `.m4a` to
/// `.m4a.failed` so recovery treats the recording as terminal. HTTP 400 is
/// the archetypal "the payload itself is wrong" response — retrying the
/// same bytes will hit the same wall, so the file is marked and only a
/// manual admin reset un-fails it.
#[tokio::test]
async fn worker_renames_audio_to_failed_on_whisper_error() {
async fn worker_renames_audio_to_failed_on_permanent_whisper_error() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/asr"))
.respond_with(ResponseTemplate::new(500).set_body_string("boom"))
.respond_with(ResponseTemplate::new(400).set_body_string("bad payload"))
.mount(&server)
.await;
@@ -321,6 +326,61 @@ async fn worker_renames_audio_to_failed_on_whisper_error() {
assert!(failed.exists(), "expected {} to exist", failed.display());
}
/// Transient Whisper errors (5xx / 408 / 429 / network) must NOT rename
/// the audio. The file stays as plain `.m4a` so the page-load heal
/// (`enqueue_pending_for_user`) picks it up automatically on the next
/// refresh — which is what happens when Minerva recovers.
#[tokio::test]
async fn worker_leaves_m4a_intact_on_transient_whisper_error() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/asr"))
.respond_with(ResponseTemplate::new(503).set_body_string("service unavailable"))
.mount(&server)
.await;
let tmp = tempfile::tempdir().unwrap();
let case_dir = tmp.path().join("dr_test/bbbb");
std::fs::create_dir_all(&case_dir).unwrap();
let audio = case_dir.join("2026-04-13T10-31-00Z.m4a");
std::fs::copy(fixture("sample.m4a"), &audio).unwrap();
let config = test_config_with_whisper(server.uri());
let (tx, rx) = transcribe::channel();
tx.send(transcribe::TranscribeJob {
audio_path: audio.clone(),
user_slug: "dr_test".into(),
})
.await
.unwrap();
drop(tx);
let client = reqwest::Client::new();
let busy = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let vocab = std::sync::Arc::new(doctate_server::gazetteer::Gazetteer::empty());
transcribe::worker::run(
rx,
config,
client,
busy,
vocab,
doctate_server::events::channel(),
)
.await;
// Original .m4a must still be there; .failed must NOT exist.
assert!(audio.exists(), "transient error must not consume the .m4a");
let failed = case_dir.join("2026-04-13T10-31-00Z.m4a.failed");
assert!(
!failed.exists(),
"transient error must not mark {} as failed",
failed.display()
);
// And no transcript yet either (the 503 produced no text).
let transcript = case_dir.join("2026-04-13T10-31-00Z.transcript.txt");
assert!(!transcript.exists(), "no transcript expected on 503");
}
/// The worker must route the Whisper response through the Gazetteer
/// before persisting. Mock returns the drift form "Zerebrum"; the
/// persisted `.transcript.txt` must contain the canonical "Cerebrum".
@@ -0,0 +1,172 @@
//! End-to-end regression test for the transient-transcribe-failure heal.
//!
//! Scenario: Minerva is briefly offline (HTTP 503) when the doctor's
//! recording arrives. The transcribe worker classifies the error as
//! transient and leaves the audio as plain `.m4a` — no `.failed`
//! suffix. On the next page-load, `heal_orphans_if_idle` re-enqueues
//! the pending `.m4a`. By then Minerva is back; Whisper returns the
//! transcript and the case advances normally.
//!
//! Pre-fix behaviour: any Whisper error renamed `.m4a` → `.m4a.failed`,
//! terminating progress permanently. Only a manual admin reset un-failed
//! the recording.
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use doctate_server::config::{Config, User};
use doctate_server::events;
use doctate_server::gazetteer::Gazetteer;
use doctate_server::{
AnalyzeBusy, OnelinerHealBusy, PipelineState, TranscribeBusy, WorkerBusy, analyze, transcribe,
};
use tempfile::tempdir;
use wiremock::matchers::{method, path as wm_path};
use wiremock::{Mock, MockServer, ResponseTemplate};
fn fixture(name: &str) -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures")
.join(name)
}
#[tokio::test]
async fn transient_whisper_failure_is_reenqueued_and_recovers() {
// Arrange filesystem: one real m4a for user "dr_test".
let data = tempdir().unwrap();
let slug = "dr_test";
let user_root = data.path().join(slug);
let case_dir = user_root.join("case-transient");
std::fs::create_dir_all(&case_dir).unwrap();
let audio = case_dir.join("2026-04-15T20-30-00Z.m4a");
std::fs::copy(fixture("sample.m4a"), &audio).unwrap();
// Arrange Whisper mock: first call → 503 (Minerva down),
// subsequent calls → 200 with a transcript. wiremock matches
// mocks in registration order; `up_to_n_times(1)` retires the
// first mock after one hit, so the second takes over for any
// retry.
let whisper = MockServer::start().await;
Mock::given(method("POST"))
.and(wm_path("/asr"))
.respond_with(ResponseTemplate::new(503).set_body_string("service unavailable"))
.up_to_n_times(1)
.mount(&whisper)
.await;
Mock::given(method("POST"))
.and(wm_path("/asr"))
.respond_with(ResponseTemplate::new(200).set_body_string("recovered text"))
.mount(&whisper)
.await;
// Config points at the mock Whisper, ollama intentionally unreachable
// (the oneliner heal may fire in parallel; we only assert on
// the transcript artefact, not on the oneliner).
let mut cfg = Config::test_default();
cfg.data_path = data.path().to_path_buf();
cfg.whisper_url = whisper.uri();
cfg.whisper_timeout_seconds = 5;
cfg.users = vec![User {
slug: slug.into(),
api_key: "k".into(),
web_password: "unused".into(),
role: "doctor".into(),
whisper: Default::default(),
retention: Default::default(),
}];
let config = Arc::new(cfg);
let vocab = Arc::new(Gazetteer::empty());
let events_tx = events::channel();
let http_client = reqwest::Client::new();
let (tx_a, _rx_a) = analyze::channel();
let (tx_t, rx_t) = transcribe::channel();
let transcribe_busy: WorkerBusy = Arc::new(AtomicBool::new(false));
let heal_busy: WorkerBusy = Arc::new(AtomicBool::new(false));
// Spawn the transcribe worker in the background; it processes jobs
// until the channel is closed at test teardown.
let worker = tokio::spawn(transcribe::worker::run(
rx_t,
config.clone(),
http_client.clone(),
transcribe_busy.clone(),
vocab.clone(),
events_tx.clone(),
));
let pipeline = PipelineState {
analyze_busy: AnalyzeBusy(Arc::new(AtomicBool::new(false))),
analyze_tx: tx_a,
transcribe_busy: TranscribeBusy(transcribe_busy.clone()),
transcribe_tx: tx_t.clone(),
oneliner_heal_busy: OnelinerHealBusy(heal_busy.clone()),
};
// Act 1: enqueue the initial job, wait for the 503 failure.
tx_t.send(transcribe::TranscribeJob {
audio_path: audio.clone(),
user_slug: slug.into(),
})
.await
.unwrap();
// Wait until the worker has processed the first job (busy false
// AND the 503 mock has recorded a hit). Polling both conditions
// avoids a race where the worker hasn't started yet.
let start = std::time::Instant::now();
loop {
let idle = !transcribe_busy.load(Ordering::Acquire);
let hits = whisper.received_requests().await.unwrap().len();
if idle && hits >= 1 {
break;
}
if start.elapsed() > Duration::from_secs(10) {
panic!("worker did not process first job within 10s (idle={idle}, hits={hits})");
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
// Invariant after transient failure: .m4a still there, no .failed.
assert!(audio.exists(), "transient error must not consume the .m4a");
assert!(
!case_dir.join("2026-04-15T20-30-00Z.m4a.failed").exists(),
".m4a.failed must not exist after transient error"
);
assert!(
!case_dir
.join("2026-04-15T20-30-00Z.transcript.txt")
.exists(),
"no transcript yet — the 503 produced nothing"
);
// Act 2: heal re-enqueues the pending .m4a (second mock → 200).
pipeline
.heal_orphans_if_idle(&user_root, slug, &http_client, &config, &vocab, &events_tx)
.await;
// Assert: the transcript lands. Poll because the worker runs
// asynchronously after the heal hands off the job.
let transcript = case_dir.join("2026-04-15T20-30-00Z.transcript.txt");
let start = std::time::Instant::now();
while !transcript.exists() {
if start.elapsed() > Duration::from_secs(10) {
panic!(
"transcript did not appear within 10s (whisper hits: {})",
whisper.received_requests().await.unwrap().len()
);
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
let body = std::fs::read_to_string(&transcript).unwrap();
assert_eq!(body, "recovered text");
// Teardown: close the worker channel so the background task exits.
drop(tx_t);
drop(pipeline);
let _ = tokio::time::timeout(Duration::from_secs(5), worker).await;
}