fix: separate transient from permanent transcribe failures
Before, every Whisper error (5xx, timeout, Minerva down, corrupt audio,
4xx) renamed `<ts>.m4a` to `<ts>.m4a.failed` uniformly, turning transient
outages into permanent sackgassen that only a manual admin reset could
undo. Cases whose every recording was `.m4a.failed` stuck without a
persisted oneliner state; the UI masked them via a fallback in
`compute_oneliner_display`.
The core insight: a recoverable error is not an error. Transient Whisper
failures now leave the audio as plain `.m4a` so the existing page-load
heal (`enqueue_pending_for_user`) re-enqueues it on the next refresh —
which is exactly what happens when Minerva comes back. No new sidecar,
no retry count, no scheduler.
Changes:
- `WhisperError::is_transient` classifies `Http`, 5xx, 408, 429 as
transient; `Io` and other 4xx as permanent.
- Transcribe worker: transient → info log + continue (audio stays .m4a);
permanent → `mark_failed` as before.
- `has_any_transcript` counts `.m4a.failed` as terminal, so
`update_oneliner` runs for failed-only cases and settles
`OnelinerState::Empty` (analogous to the silent-only fix in 4531f85).
- New verdrängender `fehler`-Badge (#c00) in case list and detail when
at least one `.m4a.failed` exists; recording-level message shortened
to plain "Transkription fehlgeschlagen".
Tests:
- New integration: transient 503 leaves `.m4a` intact, heal recovers.
- New integration: `.m4a.failed`-only case settles to
`OnelinerState::Empty` without calling Ollama.
- New unit: `is_transient` table test across relevant status codes.
- New unit: `has_any_transcript` returns true for `.m4a.failed`-only
case and false for pending-only case.
- Existing worker test retargeted from 500 to 400 and renamed; added
companion `worker_leaves_m4a_intact_on_transient_whisper_error`.
This commit is contained in:
@@ -0,0 +1,99 @@
|
||||
//! Regression test: a case whose recordings are all `.m4a.failed`
|
||||
//! (permanent transcription failures — ffmpeg corrupt audio or Whisper
|
||||
//! 4xx) must produce an `OnelinerState::Empty` automatically, so the UI
|
||||
//! has a durable terminal state instead of relying on the on-render
|
||||
//! `compute_oneliner_display` fallback.
|
||||
//!
|
||||
//! The historical bug path: `.m4a.failed`-only cases were skipped by the
|
||||
//! recovery scan (`has_any_transcript` looked for `*.transcript.txt`
|
||||
//! only), so `update_oneliner` never ran, so no `oneliner.json` was
|
||||
//! written. The UI masked the symptom with its `non_failed_count == 0`
|
||||
//! fallback — but a future refactor of that fallback would regress the
|
||||
//! case. The fix makes `has_any_transcript` count `.m4a.failed` too, so
|
||||
//! `update_oneliner` runs, sees no content transcripts, and settles the
|
||||
//! case to `Empty` through the same terminal branch used for silent-only
|
||||
//! cases.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use doctate_common::oneliners::OnelinerState;
|
||||
use doctate_server::config::Config;
|
||||
use doctate_server::events;
|
||||
use doctate_server::gazetteer::Gazetteer;
|
||||
use doctate_server::{
|
||||
AnalyzeBusy, OnelinerHealBusy, PipelineState, TranscribeBusy, WorkerBusy, analyze, transcribe,
|
||||
};
|
||||
use tempfile::tempdir;
|
||||
use wiremock::matchers::{method, path as wm_path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_only_case_settles_to_empty_without_llm_call() {
|
||||
let data = tempdir().unwrap();
|
||||
let slug = "dr_test";
|
||||
let user_root = data.path().join(slug);
|
||||
let case_dir = user_root.join("case-failed");
|
||||
std::fs::create_dir_all(&case_dir).unwrap();
|
||||
// Seed: only a `.m4a.failed` file. No transcript, no plain .m4a.
|
||||
std::fs::write(case_dir.join("2026-04-15T20-30-00Z.m4a.failed"), b"x").unwrap();
|
||||
|
||||
// Ollama must NOT be called — nothing to summarize for a failed-only
|
||||
// case. `.expect(0)` fails the test (on MockServer drop) if any
|
||||
// request arrived.
|
||||
let mock = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(wm_path("/api/chat"))
|
||||
.respond_with(ResponseTemplate::new(200))
|
||||
.expect(0)
|
||||
.mount(&mock)
|
||||
.await;
|
||||
|
||||
let mut cfg = Config::test_default();
|
||||
cfg.data_path = data.path().to_path_buf();
|
||||
cfg.ollama_url = mock.uri();
|
||||
let config = Arc::new(cfg);
|
||||
|
||||
let vocab = Arc::new(Gazetteer::empty());
|
||||
let events_tx = events::channel();
|
||||
let http_client = reqwest::Client::new();
|
||||
let (tx_a, _rx_a) = analyze::channel();
|
||||
let (tx_t, _rx_t) = transcribe::channel();
|
||||
|
||||
let heal_busy: WorkerBusy = Arc::new(AtomicBool::new(false));
|
||||
let pipeline = PipelineState {
|
||||
analyze_busy: AnalyzeBusy(Arc::new(AtomicBool::new(false))),
|
||||
analyze_tx: tx_a,
|
||||
transcribe_busy: TranscribeBusy(Arc::new(AtomicBool::new(false))),
|
||||
transcribe_tx: tx_t,
|
||||
oneliner_heal_busy: OnelinerHealBusy(heal_busy.clone()),
|
||||
};
|
||||
|
||||
pipeline
|
||||
.heal_orphans_if_idle(&user_root, slug, &http_client, &config, &vocab, &events_tx)
|
||||
.await;
|
||||
|
||||
// Heal spawn drops the busy flag on completion — poll until idle.
|
||||
let start = std::time::Instant::now();
|
||||
while heal_busy.load(Ordering::Acquire) {
|
||||
if start.elapsed() > Duration::from_secs(5) {
|
||||
panic!("oneliner heal did not finish within 5s");
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
|
||||
let path = case_dir.join("oneliner.json");
|
||||
assert!(
|
||||
path.exists(),
|
||||
"expected oneliner.json to be written for failed-only case"
|
||||
);
|
||||
let bytes = std::fs::read(&path).unwrap();
|
||||
let state: OnelinerState = serde_json::from_slice(&bytes).unwrap();
|
||||
assert!(
|
||||
matches!(state, OnelinerState::Empty { .. }),
|
||||
"expected OnelinerState::Empty, got {state:?}"
|
||||
);
|
||||
|
||||
// Mock.drop() runs here — `.expect(0)` panics if Ollama was touched.
|
||||
}
|
||||
@@ -276,12 +276,17 @@ fn test_config_with_whisper(whisper_url: String) -> Arc<Config> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Permanent Whisper errors (4xx other than 408/429) must rename `.m4a` to
|
||||
/// `.m4a.failed` so recovery treats the recording as terminal. HTTP 400 is
|
||||
/// the archetypal "the payload itself is wrong" response — retrying the
|
||||
/// same bytes will hit the same wall, so the file is marked and only a
|
||||
/// manual admin reset un-fails it.
|
||||
#[tokio::test]
|
||||
async fn worker_renames_audio_to_failed_on_whisper_error() {
|
||||
async fn worker_renames_audio_to_failed_on_permanent_whisper_error() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/asr"))
|
||||
.respond_with(ResponseTemplate::new(500).set_body_string("boom"))
|
||||
.respond_with(ResponseTemplate::new(400).set_body_string("bad payload"))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
@@ -321,6 +326,61 @@ async fn worker_renames_audio_to_failed_on_whisper_error() {
|
||||
assert!(failed.exists(), "expected {} to exist", failed.display());
|
||||
}
|
||||
|
||||
/// Transient Whisper errors (5xx / 408 / 429 / network) must NOT rename
|
||||
/// the audio. The file stays as plain `.m4a` so the page-load heal
|
||||
/// (`enqueue_pending_for_user`) picks it up automatically on the next
|
||||
/// refresh — which is what happens when Minerva recovers.
|
||||
#[tokio::test]
|
||||
async fn worker_leaves_m4a_intact_on_transient_whisper_error() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/asr"))
|
||||
.respond_with(ResponseTemplate::new(503).set_body_string("service unavailable"))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let case_dir = tmp.path().join("dr_test/bbbb");
|
||||
std::fs::create_dir_all(&case_dir).unwrap();
|
||||
let audio = case_dir.join("2026-04-13T10-31-00Z.m4a");
|
||||
std::fs::copy(fixture("sample.m4a"), &audio).unwrap();
|
||||
|
||||
let config = test_config_with_whisper(server.uri());
|
||||
let (tx, rx) = transcribe::channel();
|
||||
tx.send(transcribe::TranscribeJob {
|
||||
audio_path: audio.clone(),
|
||||
user_slug: "dr_test".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
drop(tx);
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let busy = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let vocab = std::sync::Arc::new(doctate_server::gazetteer::Gazetteer::empty());
|
||||
transcribe::worker::run(
|
||||
rx,
|
||||
config,
|
||||
client,
|
||||
busy,
|
||||
vocab,
|
||||
doctate_server::events::channel(),
|
||||
)
|
||||
.await;
|
||||
|
||||
// Original .m4a must still be there; .failed must NOT exist.
|
||||
assert!(audio.exists(), "transient error must not consume the .m4a");
|
||||
let failed = case_dir.join("2026-04-13T10-31-00Z.m4a.failed");
|
||||
assert!(
|
||||
!failed.exists(),
|
||||
"transient error must not mark {} as failed",
|
||||
failed.display()
|
||||
);
|
||||
// And no transcript yet either (the 503 produced no text).
|
||||
let transcript = case_dir.join("2026-04-13T10-31-00Z.transcript.txt");
|
||||
assert!(!transcript.exists(), "no transcript expected on 503");
|
||||
}
|
||||
|
||||
/// The worker must route the Whisper response through the Gazetteer
|
||||
/// before persisting. Mock returns the drift form "Zerebrum"; the
|
||||
/// persisted `.transcript.txt` must contain the canonical "Cerebrum".
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
//! End-to-end regression test for the transient-transcribe-failure heal.
|
||||
//!
|
||||
//! Scenario: Minerva is briefly offline (HTTP 503) when the doctor's
|
||||
//! recording arrives. The transcribe worker classifies the error as
|
||||
//! transient and leaves the audio as plain `.m4a` — no `.failed`
|
||||
//! suffix. On the next page-load, `heal_orphans_if_idle` re-enqueues
|
||||
//! the pending `.m4a`. By then Minerva is back; Whisper returns the
|
||||
//! transcript and the case advances normally.
|
||||
//!
|
||||
//! Pre-fix behaviour: any Whisper error renamed `.m4a` → `.m4a.failed`,
|
||||
//! terminating progress permanently. Only a manual admin reset un-failed
|
||||
//! the recording.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use doctate_server::config::{Config, User};
|
||||
use doctate_server::events;
|
||||
use doctate_server::gazetteer::Gazetteer;
|
||||
use doctate_server::{
|
||||
AnalyzeBusy, OnelinerHealBusy, PipelineState, TranscribeBusy, WorkerBusy, analyze, transcribe,
|
||||
};
|
||||
use tempfile::tempdir;
|
||||
use wiremock::matchers::{method, path as wm_path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
fn fixture(name: &str) -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures")
|
||||
.join(name)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transient_whisper_failure_is_reenqueued_and_recovers() {
|
||||
// Arrange filesystem: one real m4a for user "dr_test".
|
||||
let data = tempdir().unwrap();
|
||||
let slug = "dr_test";
|
||||
let user_root = data.path().join(slug);
|
||||
let case_dir = user_root.join("case-transient");
|
||||
std::fs::create_dir_all(&case_dir).unwrap();
|
||||
let audio = case_dir.join("2026-04-15T20-30-00Z.m4a");
|
||||
std::fs::copy(fixture("sample.m4a"), &audio).unwrap();
|
||||
|
||||
// Arrange Whisper mock: first call → 503 (Minerva down),
|
||||
// subsequent calls → 200 with a transcript. wiremock matches
|
||||
// mocks in registration order; `up_to_n_times(1)` retires the
|
||||
// first mock after one hit, so the second takes over for any
|
||||
// retry.
|
||||
let whisper = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(wm_path("/asr"))
|
||||
.respond_with(ResponseTemplate::new(503).set_body_string("service unavailable"))
|
||||
.up_to_n_times(1)
|
||||
.mount(&whisper)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(wm_path("/asr"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_string("recovered text"))
|
||||
.mount(&whisper)
|
||||
.await;
|
||||
|
||||
// Config points at the mock Whisper, ollama intentionally unreachable
|
||||
// (the oneliner heal may fire in parallel; we only assert on
|
||||
// the transcript artefact, not on the oneliner).
|
||||
let mut cfg = Config::test_default();
|
||||
cfg.data_path = data.path().to_path_buf();
|
||||
cfg.whisper_url = whisper.uri();
|
||||
cfg.whisper_timeout_seconds = 5;
|
||||
cfg.users = vec![User {
|
||||
slug: slug.into(),
|
||||
api_key: "k".into(),
|
||||
web_password: "unused".into(),
|
||||
role: "doctor".into(),
|
||||
whisper: Default::default(),
|
||||
retention: Default::default(),
|
||||
}];
|
||||
let config = Arc::new(cfg);
|
||||
|
||||
let vocab = Arc::new(Gazetteer::empty());
|
||||
let events_tx = events::channel();
|
||||
let http_client = reqwest::Client::new();
|
||||
|
||||
let (tx_a, _rx_a) = analyze::channel();
|
||||
let (tx_t, rx_t) = transcribe::channel();
|
||||
let transcribe_busy: WorkerBusy = Arc::new(AtomicBool::new(false));
|
||||
let heal_busy: WorkerBusy = Arc::new(AtomicBool::new(false));
|
||||
|
||||
// Spawn the transcribe worker in the background; it processes jobs
|
||||
// until the channel is closed at test teardown.
|
||||
let worker = tokio::spawn(transcribe::worker::run(
|
||||
rx_t,
|
||||
config.clone(),
|
||||
http_client.clone(),
|
||||
transcribe_busy.clone(),
|
||||
vocab.clone(),
|
||||
events_tx.clone(),
|
||||
));
|
||||
|
||||
let pipeline = PipelineState {
|
||||
analyze_busy: AnalyzeBusy(Arc::new(AtomicBool::new(false))),
|
||||
analyze_tx: tx_a,
|
||||
transcribe_busy: TranscribeBusy(transcribe_busy.clone()),
|
||||
transcribe_tx: tx_t.clone(),
|
||||
oneliner_heal_busy: OnelinerHealBusy(heal_busy.clone()),
|
||||
};
|
||||
|
||||
// Act 1: enqueue the initial job, wait for the 503 failure.
|
||||
tx_t.send(transcribe::TranscribeJob {
|
||||
audio_path: audio.clone(),
|
||||
user_slug: slug.into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Wait until the worker has processed the first job (busy false
|
||||
// AND the 503 mock has recorded a hit). Polling both conditions
|
||||
// avoids a race where the worker hasn't started yet.
|
||||
let start = std::time::Instant::now();
|
||||
loop {
|
||||
let idle = !transcribe_busy.load(Ordering::Acquire);
|
||||
let hits = whisper.received_requests().await.unwrap().len();
|
||||
if idle && hits >= 1 {
|
||||
break;
|
||||
}
|
||||
if start.elapsed() > Duration::from_secs(10) {
|
||||
panic!("worker did not process first job within 10s (idle={idle}, hits={hits})");
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
|
||||
// Invariant after transient failure: .m4a still there, no .failed.
|
||||
assert!(audio.exists(), "transient error must not consume the .m4a");
|
||||
assert!(
|
||||
!case_dir.join("2026-04-15T20-30-00Z.m4a.failed").exists(),
|
||||
".m4a.failed must not exist after transient error"
|
||||
);
|
||||
assert!(
|
||||
!case_dir
|
||||
.join("2026-04-15T20-30-00Z.transcript.txt")
|
||||
.exists(),
|
||||
"no transcript yet — the 503 produced nothing"
|
||||
);
|
||||
|
||||
// Act 2: heal re-enqueues the pending .m4a (second mock → 200).
|
||||
pipeline
|
||||
.heal_orphans_if_idle(&user_root, slug, &http_client, &config, &vocab, &events_tx)
|
||||
.await;
|
||||
|
||||
// Assert: the transcript lands. Poll because the worker runs
|
||||
// asynchronously after the heal hands off the job.
|
||||
let transcript = case_dir.join("2026-04-15T20-30-00Z.transcript.txt");
|
||||
let start = std::time::Instant::now();
|
||||
while !transcript.exists() {
|
||||
if start.elapsed() > Duration::from_secs(10) {
|
||||
panic!(
|
||||
"transcript did not appear within 10s (whisper hits: {})",
|
||||
whisper.received_requests().await.unwrap().len()
|
||||
);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
|
||||
let body = std::fs::read_to_string(&transcript).unwrap();
|
||||
assert_eq!(body, "recovered text");
|
||||
|
||||
// Teardown: close the worker channel so the background task exits.
|
||||
drop(tx_t);
|
||||
drop(pipeline);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), worker).await;
|
||||
}
|
||||
Reference in New Issue
Block a user