From 1f32d4dd23184d7925b3abe444dc9e85b5fd4a61 Mon Sep 17 00:00:00 2001 From: Brummel Date: Tue, 21 Apr 2026 14:39:31 +0200 Subject: [PATCH] fix: separate transient from permanent transcribe failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before, every Whisper error (5xx, timeout, Minerva down, corrupt audio, 4xx) renamed `.m4a` to `.m4a.failed` uniformly, turning transient outages into permanent sackgassen that only a manual admin reset could undo. Cases whose every recording was `.m4a.failed` stuck without a persisted oneliner state; the UI masked them via a fallback in `compute_oneliner_display`. The core insight: a recoverable error is not an error. Transient Whisper failures now leave the audio as plain `.m4a` so the existing page-load heal (`enqueue_pending_for_user`) re-enqueues it on the next refresh — which is exactly what happens when Minerva comes back. No new sidecar, no retry count, no scheduler. Changes: - `WhisperError::is_transient` classifies `Http`, 5xx, 408, 429 as transient; `Io` and other 4xx as permanent. - Transcribe worker: transient → info log + continue (audio stays .m4a); permanent → `mark_failed` as before. - `has_any_transcript` counts `.m4a.failed` as terminal, so `update_oneliner` runs for failed-only cases and settles `OnelinerState::Empty` (analogous to the silent-only fix in 4531f85). - New verdrängender `fehler`-Badge (#c00) in case list and detail when at least one `.m4a.failed` exists; recording-level message shortened to plain "Transkription fehlgeschlagen". Tests: - New integration: transient 503 leaves `.m4a` intact, heal recovers. - New integration: `.m4a.failed`-only case settles to `OnelinerState::Empty` without calling Ollama. - New unit: `is_transient` table test across relevant status codes. - New unit: `has_any_transcript` returns true for `.m4a.failed`-only case and false for pending-only case. - Existing worker test retargeted from 500 to 400 and renamed; added companion `worker_leaves_m4a_intact_on_transient_whisper_error`. --- server/src/routes/user_web.rs | 14 ++ server/src/transcribe/recovery.rs | 68 ++++++- server/src/transcribe/whisper.rs | 74 ++++++++ server/src/transcribe/worker.rs | 15 +- server/templates/case_page.html | 9 +- server/templates/case_recordings.html | 2 +- server/templates/my_cases.html | 3 +- .../failed_only_case_empty_oneliner_test.rs | 99 ++++++++++ server/tests/transcribe_test.rs | 64 ++++++- .../tests/transient_failure_retries_test.rs | 172 ++++++++++++++++++ 10 files changed, 502 insertions(+), 18 deletions(-) create mode 100644 server/tests/failed_only_case_empty_oneliner_test.rs create mode 100644 server/tests/transient_failure_retries_test.rs diff --git a/server/src/routes/user_web.rs b/server/src/routes/user_web.rs index c8de913..2527f1f 100644 --- a/server/src/routes/user_web.rs +++ b/server/src/routes/user_web.rs @@ -71,6 +71,12 @@ struct UserCaseView { recordings_count: usize, analyzing: bool, has_document: bool, + /// True iff at least one recording is `.m4a.failed` — a *permanent* + /// transcribe failure (ffmpeg remux or Whisper 4xx). Transient Whisper + /// errors never set this flag because they leave the file as plain + /// `.m4a` for the page-load heal to retry. Drives the verdrängende + /// `fehler`-Badge over `offen`/`ausgewertet`. + has_failed_recording: bool, /// True iff the inline "Analysieren"-button should be enabled. /// Requires: not currently analyzing, all non-failed recordings /// transcribed, and an LLM is configured. @@ -213,6 +219,10 @@ struct CasePageTemplate { llm_missing: bool, analyzing: bool, has_document: bool, + /// True iff at least one `.m4a.failed` recording exists. Drives the + /// verdrängende `fehler`-Badge in the case header; see the same + /// field on `UserCaseView` for semantics. + has_failed_recording: bool, is_admin: bool, /// True when the case carries a `.closed` marker. Toggles the /// header action button between close and reopen, and appends @@ -491,6 +501,7 @@ pub async fn handle_case_page( .iter() .filter(|r| r.transcript.has_file()) .count(); + let has_failed_recording = recordings.iter().any(|r| r.failed); let case_id_short = case_id_str.chars().take(8).collect(); let is_admin = user.is_admin(); @@ -515,6 +526,7 @@ pub async fn handle_case_page( llm_missing: flags.llm_missing, analyzing: flags.analyzing, has_document: flags.has_document, + has_failed_recording, is_admin, is_closed, } @@ -732,6 +744,7 @@ async fn compute_case_view( .unwrap_or_default(); let recordings_count = recordings.len(); let non_failed_count = recordings.iter().filter(|r| !r.failed).count(); + let has_failed_recording = recordings.iter().any(|r| r.failed); let all_transcribed = non_failed_count > 0 && recordings .iter() @@ -757,6 +770,7 @@ async fn compute_case_view( recordings_count, analyzing, has_document, + has_failed_recording, can_analyze, is_closed, days_until_purge, diff --git a/server/src/transcribe/recovery.rs b/server/src/transcribe/recovery.rs index 96a0854..0294781 100644 --- a/server/src/transcribe/recovery.rs +++ b/server/src/transcribe/recovery.rs @@ -156,16 +156,26 @@ pub(crate) async fn cases_needing_oneliner_in(user_root: &Path) -> Vec out } -/// True iff at least one `.transcript.txt` sidecar exists in `case_dir` -/// — i.e., at least one recording is either [`TranscriptState::Silent`] -/// or [`TranscriptState::Content`]. Cases where every recording is -/// still `Pending` return false: there's nothing for the oneliner -/// worker to act on yet. +/// True iff the case has at least one recording in a *terminal* state — +/// meaning the transcribe pipeline will produce no further artefacts +/// and `update_oneliner` has enough information to settle the case. /// -/// Silent transcripts count here because `update_oneliner` needs to -/// settle them to `OnelinerState::Empty`. The previous implementation -/// required *content* specifically, which left silent-only cases stuck -/// with no oneliner state — and the UI stuck on "Generating". +/// Three terminal shapes count: +/// - `.transcript.txt` with content → [`TranscriptState::Content`] +/// - `.transcript.txt` empty → [`TranscriptState::Silent`] +/// - `.m4a.failed` → permanent transcription failure (ffmpeg remux or +/// Whisper 4xx). Transient Whisper errors leave the file as plain +/// `.m4a` without sidecar (Pending), so they do *not* match here — +/// the page-load heal re-enqueues them on the next refresh. +/// +/// Cases where every recording is still `Pending` (plain `.m4a`, no +/// sidecar, not marked failed) return false: the transcriber hasn't +/// finished yet and there is nothing for the oneliner worker to act +/// on. +/// +/// Both Silent and permanent-failed count because `update_oneliner` +/// settles each to [`OnelinerState::Empty`] via the same no-content +/// branch in `worker::update_oneliner`. async fn has_any_transcript(case_dir: &Path) -> bool { let Ok(mut files) = tokio::fs::read_dir(case_dir).await else { return false; @@ -173,7 +183,7 @@ async fn has_any_transcript(case_dir: &Path) -> bool { while let Ok(Some(f)) = files.next_entry().await { if f.file_name() .to_str() - .is_some_and(|s| s.ends_with(".transcript.txt")) + .is_some_and(|s| s.ends_with(".transcript.txt") || s.ends_with(".m4a.failed")) { return true; } @@ -356,4 +366,42 @@ mod tests { assert!(cases_needing_oneliner(data.path()).await.is_empty()); } + + /// `.m4a.failed`-only cases are terminal — the pipeline will produce + /// no further artefacts. Recovery must pick them up so + /// `update_oneliner` can persist `OnelinerState::Empty`; otherwise + /// the case would rely on the UI fallback forever and never have a + /// stable on-disk state. + #[tokio::test] + async fn cases_needing_oneliner_picks_up_failed_only_cases() { + let data = tempdir().unwrap(); + let case = data.path().join("user").join("case1"); + tokio::fs::create_dir_all(&case).await.unwrap(); + tokio::fs::write(case.join("2026-04-16T10-00-00Z.m4a.failed"), b"x") + .await + .unwrap(); + + let got = cases_needing_oneliner(data.path()).await; + assert_eq!(got, vec![(case, "user".to_owned())]); + } + + #[tokio::test] + async fn has_any_transcript_true_for_failed_only_case() { + let case = tempdir().unwrap(); + tokio::fs::write(case.path().join("2026-04-16T10-00-00Z.m4a.failed"), b"x") + .await + .unwrap(); + + assert!(has_any_transcript(case.path()).await); + } + + #[tokio::test] + async fn has_any_transcript_false_for_pending_only_case() { + let case = tempdir().unwrap(); + tokio::fs::write(case.path().join("2026-04-16T10-00-00Z.m4a"), b"audio") + .await + .unwrap(); + + assert!(!has_any_transcript(case.path()).await); + } } diff --git a/server/src/transcribe/whisper.rs b/server/src/transcribe/whisper.rs index be8ec55..d8fea3d 100644 --- a/server/src/transcribe/whisper.rs +++ b/server/src/transcribe/whisper.rs @@ -27,6 +27,35 @@ impl std::fmt::Display for WhisperError { impl std::error::Error for WhisperError {} +impl WhisperError { + /// Classify this error as transient (retry will likely succeed later) or + /// permanent (retrying the same input will hit the same wall). + /// + /// Rationale: the transcribe worker uses this to decide whether to leave + /// the `.m4a` in place (page-load heal re-enqueues it) or rename it to + /// `.m4a.failed` (terminal; only a manual reset un-fails it). + /// + /// - `Http`: reqwest wraps all transport issues (DNS, connect, TLS, read, + /// timeout, mid-flight reset) — every one of these is a candidate for + /// „Minerva is back in a minute", so treat as transient. + /// - `Status`: 5xx is server-side trouble; 408 (Request Timeout) and 429 + /// (Too Many Requests) are explicit retry signals. Everything else in + /// the 4xx range (400 Bad Request, 415 Unsupported Media, ...) means + /// the payload itself is the problem and replays will fail identically. + /// - `Io`: local filesystem error while reading the audio file. Either + /// the file is gone or the disk is misbehaving; a retry of the same + /// path is not the right response. + pub fn is_transient(&self) -> bool { + match self { + Self::Http(_) => true, + Self::Status { status, .. } => { + *status == 408 || *status == 429 || (500..=599).contains(status) + } + Self::Io(_) => false, + } + } +} + /// POST the audio file to whisper-asr-webservice and return the transcript text. /// /// `output=txt` → plain-text response. `language` pins the language so the model @@ -97,3 +126,48 @@ pub async fn transcribe( Ok(body) } + +#[cfg(test)] +mod tests { + use super::*; + + /// Table-driven classification test. `Http` variant is not publicly + /// constructible (reqwest owns its error ctor), so the transient-Http + /// path is covered by the integration test in + /// `tests/transient_failure_retries_test.rs`. Same code path is exercised + /// here via a 503 Status, which also routes to `is_transient == true`. + #[test] + fn is_transient_classifies_status_codes() { + let cases: &[(u16, bool)] = &[ + (400, false), + (401, false), + (403, false), + (404, false), + (408, true), + (415, false), + (422, false), + (429, true), + (500, true), + (502, true), + (503, true), + (504, true), + ]; + for &(status, expected) in cases { + let err = WhisperError::Status { + status, + body: String::new(), + }; + assert_eq!( + err.is_transient(), + expected, + "status {status} expected is_transient={expected}" + ); + } + } + + #[test] + fn is_transient_io_is_permanent() { + let err = WhisperError::Io(std::io::Error::other("disk gone")); + assert!(!err.is_transient()); + } +} diff --git a/server/src/transcribe/worker.rs b/server/src/transcribe/worker.rs index bd07d88..79fee95 100644 --- a/server/src/transcribe/worker.rs +++ b/server/src/transcribe/worker.rs @@ -78,7 +78,20 @@ pub async fn run( Ok(t) => t, Err(e) => { error!(audio = %audio_path.display(), error = %e, "whisper call failed"); - mark_failed(&audio_path, &events_tx, &job.user_slug).await; + if e.is_transient() { + // Leave the recording as plain `.m4a` (no `.failed` suffix) + // so the page-load heal (`enqueue_pending_for_user`) picks + // it up on the next refresh. The natural retry trigger is + // a human navigating the UI — no scheduler, no sidecar. + info!( + audio = %audio_path.display(), + error = %e, + reason = "transient", + "recording left as .m4a, page-load heal will re-enqueue" + ); + } else { + mark_failed(&audio_path, &events_tx, &job.user_slug).await; + } continue; } }; diff --git a/server/templates/case_page.html b/server/templates/case_page.html index eb1ba6e..1f66aed 100644 --- a/server/templates/case_page.html +++ b/server/templates/case_page.html @@ -67,6 +67,9 @@ .status-badge.done { background: #7ed321; } + .status-badge.fehler { + background: #c00; + } .actions { margin: 1em 0 1.5em; display: flex; @@ -255,9 +258,9 @@

- {% call ol::render(oneliner) %} {% if has_document %}ausgewertetfehler{% else if + has_document %}ausgewertet{% else %}offen{% endif %} diff --git a/server/templates/case_recordings.html b/server/templates/case_recordings.html index b62a2f0..b79df7f 100644 --- a/server/templates/case_recordings.html +++ b/server/templates/case_recordings.html @@ -114,7 +114,7 @@ try { {% if is_admin %}
{{ rec.filename }}
{% endif %} {% if rec.failed %} -
Transkription fehlgeschlagen — .failed-Suffix entfernen zum Erneut-Versuchen.
+
Transkription fehlgeschlagen
{% else %} {% match rec.transcript %} {% when TranscriptState::Content with (t) %} diff --git a/server/templates/my_cases.html b/server/templates/my_cases.html index 2c999fe..938c0e1 100644 --- a/server/templates/my_cases.html +++ b/server/templates/my_cases.html @@ -32,6 +32,7 @@ section h2 .count { font-weight: normal; font-size: 0.85em; color: #888; } .status-badge { display: inline-block; padding: 0.05em 0.5em; border-radius: 999px; font-size: 0.75em; font-weight: bold; color: white; } .status-badge.open { background: #e24a4a; } .status-badge.done { background: #7ed321; } +.status-badge.fehler { background: #c00; } .case-row .actions { display: flex; gap: 0.4em; align-items: center; } .case-row .actions button { font-size: 0.85em; padding: 0.35em 0.8em; } .case-row .actions .delete-btn { padding: 0.3em; background: transparent; border: none; color: #888; cursor: pointer; display: inline-flex; align-items: center; line-height: 0; border-radius: 4px; transition: color 0.15s, background 0.15s; } @@ -96,7 +97,7 @@ try {
— {% call ol::render(case.oneliner) %}
-
{{ case.recordings_count }} Aufnahmen{% if case.has_document %} — ausgewertet{% else if !case.is_closed %} — offen{% endif %}{% if case.analyzing %} wird analysiert{% endif %}{% if case.is_closed %}geschlossen{% match case.days_until_purge %}{% when Some with (d) %} — wird in {{ d }} Tagen entfernt{% when None %}{% endmatch %}{% endif %}
+
{{ case.recordings_count }} Aufnahmen{% if case.has_failed_recording %} — fehler{% else if case.has_document %} — ausgewertet{% else if !case.is_closed %} — offen{% endif %}{% if case.analyzing %} wird analysiert{% endif %}{% if case.is_closed %}geschlossen{% match case.days_until_purge %}{% when Some with (d) %} — wird in {{ d }} Tagen entfernt{% when None %}{% endmatch %}{% endif %}
{% if is_admin %}
{{ case.case_id }}
{% endif %}
diff --git a/server/tests/failed_only_case_empty_oneliner_test.rs b/server/tests/failed_only_case_empty_oneliner_test.rs new file mode 100644 index 0000000..d3d9f8e --- /dev/null +++ b/server/tests/failed_only_case_empty_oneliner_test.rs @@ -0,0 +1,99 @@ +//! Regression test: a case whose recordings are all `.m4a.failed` +//! (permanent transcription failures — ffmpeg corrupt audio or Whisper +//! 4xx) must produce an `OnelinerState::Empty` automatically, so the UI +//! has a durable terminal state instead of relying on the on-render +//! `compute_oneliner_display` fallback. +//! +//! The historical bug path: `.m4a.failed`-only cases were skipped by the +//! recovery scan (`has_any_transcript` looked for `*.transcript.txt` +//! only), so `update_oneliner` never ran, so no `oneliner.json` was +//! written. The UI masked the symptom with its `non_failed_count == 0` +//! fallback — but a future refactor of that fallback would regress the +//! case. The fix makes `has_any_transcript` count `.m4a.failed` too, so +//! `update_oneliner` runs, sees no content transcripts, and settles the +//! case to `Empty` through the same terminal branch used for silent-only +//! cases. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use doctate_common::oneliners::OnelinerState; +use doctate_server::config::Config; +use doctate_server::events; +use doctate_server::gazetteer::Gazetteer; +use doctate_server::{ + AnalyzeBusy, OnelinerHealBusy, PipelineState, TranscribeBusy, WorkerBusy, analyze, transcribe, +}; +use tempfile::tempdir; +use wiremock::matchers::{method, path as wm_path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +#[tokio::test] +async fn failed_only_case_settles_to_empty_without_llm_call() { + let data = tempdir().unwrap(); + let slug = "dr_test"; + let user_root = data.path().join(slug); + let case_dir = user_root.join("case-failed"); + std::fs::create_dir_all(&case_dir).unwrap(); + // Seed: only a `.m4a.failed` file. No transcript, no plain .m4a. + std::fs::write(case_dir.join("2026-04-15T20-30-00Z.m4a.failed"), b"x").unwrap(); + + // Ollama must NOT be called — nothing to summarize for a failed-only + // case. `.expect(0)` fails the test (on MockServer drop) if any + // request arrived. + let mock = MockServer::start().await; + Mock::given(method("POST")) + .and(wm_path("/api/chat")) + .respond_with(ResponseTemplate::new(200)) + .expect(0) + .mount(&mock) + .await; + + let mut cfg = Config::test_default(); + cfg.data_path = data.path().to_path_buf(); + cfg.ollama_url = mock.uri(); + let config = Arc::new(cfg); + + let vocab = Arc::new(Gazetteer::empty()); + let events_tx = events::channel(); + let http_client = reqwest::Client::new(); + let (tx_a, _rx_a) = analyze::channel(); + let (tx_t, _rx_t) = transcribe::channel(); + + let heal_busy: WorkerBusy = Arc::new(AtomicBool::new(false)); + let pipeline = PipelineState { + analyze_busy: AnalyzeBusy(Arc::new(AtomicBool::new(false))), + analyze_tx: tx_a, + transcribe_busy: TranscribeBusy(Arc::new(AtomicBool::new(false))), + transcribe_tx: tx_t, + oneliner_heal_busy: OnelinerHealBusy(heal_busy.clone()), + }; + + pipeline + .heal_orphans_if_idle(&user_root, slug, &http_client, &config, &vocab, &events_tx) + .await; + + // Heal spawn drops the busy flag on completion — poll until idle. + let start = std::time::Instant::now(); + while heal_busy.load(Ordering::Acquire) { + if start.elapsed() > Duration::from_secs(5) { + panic!("oneliner heal did not finish within 5s"); + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + + let path = case_dir.join("oneliner.json"); + assert!( + path.exists(), + "expected oneliner.json to be written for failed-only case" + ); + let bytes = std::fs::read(&path).unwrap(); + let state: OnelinerState = serde_json::from_slice(&bytes).unwrap(); + assert!( + matches!(state, OnelinerState::Empty { .. }), + "expected OnelinerState::Empty, got {state:?}" + ); + + // Mock.drop() runs here — `.expect(0)` panics if Ollama was touched. +} diff --git a/server/tests/transcribe_test.rs b/server/tests/transcribe_test.rs index 36ec9fa..6c1448a 100644 --- a/server/tests/transcribe_test.rs +++ b/server/tests/transcribe_test.rs @@ -276,12 +276,17 @@ fn test_config_with_whisper(whisper_url: String) -> Arc { }) } +/// Permanent Whisper errors (4xx other than 408/429) must rename `.m4a` to +/// `.m4a.failed` so recovery treats the recording as terminal. HTTP 400 is +/// the archetypal "the payload itself is wrong" response — retrying the +/// same bytes will hit the same wall, so the file is marked and only a +/// manual admin reset un-fails it. #[tokio::test] -async fn worker_renames_audio_to_failed_on_whisper_error() { +async fn worker_renames_audio_to_failed_on_permanent_whisper_error() { let server = MockServer::start().await; Mock::given(method("POST")) .and(path("/asr")) - .respond_with(ResponseTemplate::new(500).set_body_string("boom")) + .respond_with(ResponseTemplate::new(400).set_body_string("bad payload")) .mount(&server) .await; @@ -321,6 +326,61 @@ async fn worker_renames_audio_to_failed_on_whisper_error() { assert!(failed.exists(), "expected {} to exist", failed.display()); } +/// Transient Whisper errors (5xx / 408 / 429 / network) must NOT rename +/// the audio. The file stays as plain `.m4a` so the page-load heal +/// (`enqueue_pending_for_user`) picks it up automatically on the next +/// refresh — which is what happens when Minerva recovers. +#[tokio::test] +async fn worker_leaves_m4a_intact_on_transient_whisper_error() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/asr")) + .respond_with(ResponseTemplate::new(503).set_body_string("service unavailable")) + .mount(&server) + .await; + + let tmp = tempfile::tempdir().unwrap(); + let case_dir = tmp.path().join("dr_test/bbbb"); + std::fs::create_dir_all(&case_dir).unwrap(); + let audio = case_dir.join("2026-04-13T10-31-00Z.m4a"); + std::fs::copy(fixture("sample.m4a"), &audio).unwrap(); + + let config = test_config_with_whisper(server.uri()); + let (tx, rx) = transcribe::channel(); + tx.send(transcribe::TranscribeJob { + audio_path: audio.clone(), + user_slug: "dr_test".into(), + }) + .await + .unwrap(); + drop(tx); + + let client = reqwest::Client::new(); + let busy = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let vocab = std::sync::Arc::new(doctate_server::gazetteer::Gazetteer::empty()); + transcribe::worker::run( + rx, + config, + client, + busy, + vocab, + doctate_server::events::channel(), + ) + .await; + + // Original .m4a must still be there; .failed must NOT exist. + assert!(audio.exists(), "transient error must not consume the .m4a"); + let failed = case_dir.join("2026-04-13T10-31-00Z.m4a.failed"); + assert!( + !failed.exists(), + "transient error must not mark {} as failed", + failed.display() + ); + // And no transcript yet either (the 503 produced no text). + let transcript = case_dir.join("2026-04-13T10-31-00Z.transcript.txt"); + assert!(!transcript.exists(), "no transcript expected on 503"); +} + /// The worker must route the Whisper response through the Gazetteer /// before persisting. Mock returns the drift form "Zerebrum"; the /// persisted `.transcript.txt` must contain the canonical "Cerebrum". diff --git a/server/tests/transient_failure_retries_test.rs b/server/tests/transient_failure_retries_test.rs new file mode 100644 index 0000000..795f199 --- /dev/null +++ b/server/tests/transient_failure_retries_test.rs @@ -0,0 +1,172 @@ +//! End-to-end regression test for the transient-transcribe-failure heal. +//! +//! Scenario: Minerva is briefly offline (HTTP 503) when the doctor's +//! recording arrives. The transcribe worker classifies the error as +//! transient and leaves the audio as plain `.m4a` — no `.failed` +//! suffix. On the next page-load, `heal_orphans_if_idle` re-enqueues +//! the pending `.m4a`. By then Minerva is back; Whisper returns the +//! transcript and the case advances normally. +//! +//! Pre-fix behaviour: any Whisper error renamed `.m4a` → `.m4a.failed`, +//! terminating progress permanently. Only a manual admin reset un-failed +//! the recording. + +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use doctate_server::config::{Config, User}; +use doctate_server::events; +use doctate_server::gazetteer::Gazetteer; +use doctate_server::{ + AnalyzeBusy, OnelinerHealBusy, PipelineState, TranscribeBusy, WorkerBusy, analyze, transcribe, +}; +use tempfile::tempdir; +use wiremock::matchers::{method, path as wm_path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +fn fixture(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures") + .join(name) +} + +#[tokio::test] +async fn transient_whisper_failure_is_reenqueued_and_recovers() { + // Arrange filesystem: one real m4a for user "dr_test". + let data = tempdir().unwrap(); + let slug = "dr_test"; + let user_root = data.path().join(slug); + let case_dir = user_root.join("case-transient"); + std::fs::create_dir_all(&case_dir).unwrap(); + let audio = case_dir.join("2026-04-15T20-30-00Z.m4a"); + std::fs::copy(fixture("sample.m4a"), &audio).unwrap(); + + // Arrange Whisper mock: first call → 503 (Minerva down), + // subsequent calls → 200 with a transcript. wiremock matches + // mocks in registration order; `up_to_n_times(1)` retires the + // first mock after one hit, so the second takes over for any + // retry. + let whisper = MockServer::start().await; + Mock::given(method("POST")) + .and(wm_path("/asr")) + .respond_with(ResponseTemplate::new(503).set_body_string("service unavailable")) + .up_to_n_times(1) + .mount(&whisper) + .await; + Mock::given(method("POST")) + .and(wm_path("/asr")) + .respond_with(ResponseTemplate::new(200).set_body_string("recovered text")) + .mount(&whisper) + .await; + + // Config points at the mock Whisper, ollama intentionally unreachable + // (the oneliner heal may fire in parallel; we only assert on + // the transcript artefact, not on the oneliner). + let mut cfg = Config::test_default(); + cfg.data_path = data.path().to_path_buf(); + cfg.whisper_url = whisper.uri(); + cfg.whisper_timeout_seconds = 5; + cfg.users = vec![User { + slug: slug.into(), + api_key: "k".into(), + web_password: "unused".into(), + role: "doctor".into(), + whisper: Default::default(), + retention: Default::default(), + }]; + let config = Arc::new(cfg); + + let vocab = Arc::new(Gazetteer::empty()); + let events_tx = events::channel(); + let http_client = reqwest::Client::new(); + + let (tx_a, _rx_a) = analyze::channel(); + let (tx_t, rx_t) = transcribe::channel(); + let transcribe_busy: WorkerBusy = Arc::new(AtomicBool::new(false)); + let heal_busy: WorkerBusy = Arc::new(AtomicBool::new(false)); + + // Spawn the transcribe worker in the background; it processes jobs + // until the channel is closed at test teardown. + let worker = tokio::spawn(transcribe::worker::run( + rx_t, + config.clone(), + http_client.clone(), + transcribe_busy.clone(), + vocab.clone(), + events_tx.clone(), + )); + + let pipeline = PipelineState { + analyze_busy: AnalyzeBusy(Arc::new(AtomicBool::new(false))), + analyze_tx: tx_a, + transcribe_busy: TranscribeBusy(transcribe_busy.clone()), + transcribe_tx: tx_t.clone(), + oneliner_heal_busy: OnelinerHealBusy(heal_busy.clone()), + }; + + // Act 1: enqueue the initial job, wait for the 503 failure. + tx_t.send(transcribe::TranscribeJob { + audio_path: audio.clone(), + user_slug: slug.into(), + }) + .await + .unwrap(); + + // Wait until the worker has processed the first job (busy false + // AND the 503 mock has recorded a hit). Polling both conditions + // avoids a race where the worker hasn't started yet. + let start = std::time::Instant::now(); + loop { + let idle = !transcribe_busy.load(Ordering::Acquire); + let hits = whisper.received_requests().await.unwrap().len(); + if idle && hits >= 1 { + break; + } + if start.elapsed() > Duration::from_secs(10) { + panic!("worker did not process first job within 10s (idle={idle}, hits={hits})"); + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + + // Invariant after transient failure: .m4a still there, no .failed. + assert!(audio.exists(), "transient error must not consume the .m4a"); + assert!( + !case_dir.join("2026-04-15T20-30-00Z.m4a.failed").exists(), + ".m4a.failed must not exist after transient error" + ); + assert!( + !case_dir + .join("2026-04-15T20-30-00Z.transcript.txt") + .exists(), + "no transcript yet — the 503 produced nothing" + ); + + // Act 2: heal re-enqueues the pending .m4a (second mock → 200). + pipeline + .heal_orphans_if_idle(&user_root, slug, &http_client, &config, &vocab, &events_tx) + .await; + + // Assert: the transcript lands. Poll because the worker runs + // asynchronously after the heal hands off the job. + let transcript = case_dir.join("2026-04-15T20-30-00Z.transcript.txt"); + let start = std::time::Instant::now(); + while !transcript.exists() { + if start.elapsed() > Duration::from_secs(10) { + panic!( + "transcript did not appear within 10s (whisper hits: {})", + whisper.received_requests().await.unwrap().len() + ); + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + + let body = std::fs::read_to_string(&transcript).unwrap(); + assert_eq!(body, "recovered text"); + + // Teardown: close the worker channel so the background task exits. + drop(tx_t); + drop(pipeline); + let _ = tokio::time::timeout(Duration::from_secs(5), worker).await; +}