fix: separate transient from permanent transcribe failures

Before, every Whisper error (5xx, timeout, Minerva down, corrupt audio,
4xx) renamed `<ts>.m4a` to `<ts>.m4a.failed` uniformly, turning transient
outages into permanent sackgassen that only a manual admin reset could
undo. Cases whose every recording was `.m4a.failed` stuck without a
persisted oneliner state; the UI masked them via a fallback in
`compute_oneliner_display`.

The core insight: a recoverable error is not an error. Transient Whisper
failures now leave the audio as plain `.m4a` so the existing page-load
heal (`enqueue_pending_for_user`) re-enqueues it on the next refresh —
which is exactly what happens when Minerva comes back. No new sidecar,
no retry count, no scheduler.

Changes:
- `WhisperError::is_transient` classifies `Http`, 5xx, 408, 429 as
  transient; `Io` and other 4xx as permanent.
- Transcribe worker: transient → info log + continue (audio stays .m4a);
  permanent → `mark_failed` as before.
- `has_any_transcript` counts `.m4a.failed` as terminal, so
  `update_oneliner` runs for failed-only cases and settles
  `OnelinerState::Empty` (analogous to the silent-only fix in 4531f85).
- New verdrängender `fehler`-Badge (#c00) in case list and detail when
  at least one `.m4a.failed` exists; recording-level message shortened
  to plain "Transkription fehlgeschlagen".

Tests:
- New integration: transient 503 leaves `.m4a` intact, heal recovers.
- New integration: `.m4a.failed`-only case settles to
  `OnelinerState::Empty` without calling Ollama.
- New unit: `is_transient` table test across relevant status codes.
- New unit: `has_any_transcript` returns true for `.m4a.failed`-only
  case and false for pending-only case.
- Existing worker test retargeted from 500 to 400 and renamed; added
  companion `worker_leaves_m4a_intact_on_transient_whisper_error`.
This commit is contained in:
2026-04-21 14:39:31 +02:00
parent 4531f85b13
commit 1f32d4dd23
10 changed files with 502 additions and 18 deletions
+14
View File
@@ -71,6 +71,12 @@ struct UserCaseView {
recordings_count: usize,
analyzing: bool,
has_document: bool,
/// True iff at least one recording is `.m4a.failed` — a *permanent*
/// transcribe failure (ffmpeg remux or Whisper 4xx). Transient Whisper
/// errors never set this flag because they leave the file as plain
/// `.m4a` for the page-load heal to retry. Drives the verdrängende
/// `fehler`-Badge over `offen`/`ausgewertet`.
has_failed_recording: bool,
/// True iff the inline "Analysieren"-button should be enabled.
/// Requires: not currently analyzing, all non-failed recordings
/// transcribed, and an LLM is configured.
@@ -213,6 +219,10 @@ struct CasePageTemplate {
llm_missing: bool,
analyzing: bool,
has_document: bool,
/// True iff at least one `.m4a.failed` recording exists. Drives the
/// verdrängende `fehler`-Badge in the case header; see the same
/// field on `UserCaseView` for semantics.
has_failed_recording: bool,
is_admin: bool,
/// True when the case carries a `.closed` marker. Toggles the
/// header action button between close and reopen, and appends
@@ -491,6 +501,7 @@ pub async fn handle_case_page(
.iter()
.filter(|r| r.transcript.has_file())
.count();
let has_failed_recording = recordings.iter().any(|r| r.failed);
let case_id_short = case_id_str.chars().take(8).collect();
let is_admin = user.is_admin();
@@ -515,6 +526,7 @@ pub async fn handle_case_page(
llm_missing: flags.llm_missing,
analyzing: flags.analyzing,
has_document: flags.has_document,
has_failed_recording,
is_admin,
is_closed,
}
@@ -732,6 +744,7 @@ async fn compute_case_view(
.unwrap_or_default();
let recordings_count = recordings.len();
let non_failed_count = recordings.iter().filter(|r| !r.failed).count();
let has_failed_recording = recordings.iter().any(|r| r.failed);
let all_transcribed = non_failed_count > 0
&& recordings
.iter()
@@ -757,6 +770,7 @@ async fn compute_case_view(
recordings_count,
analyzing,
has_document,
has_failed_recording,
can_analyze,
is_closed,
days_until_purge,
+58 -10
View File
@@ -156,16 +156,26 @@ pub(crate) async fn cases_needing_oneliner_in(user_root: &Path) -> Vec<PathBuf>
out
}
/// True iff at least one `.transcript.txt` sidecar exists in `case_dir`
/// — i.e., at least one recording is either [`TranscriptState::Silent`]
/// or [`TranscriptState::Content`]. Cases where every recording is
/// still `Pending` return false: there's nothing for the oneliner
/// worker to act on yet.
/// True iff the case has at least one recording in a *terminal* state —
/// meaning the transcribe pipeline will produce no further artefacts
/// and `update_oneliner` has enough information to settle the case.
///
/// Silent transcripts count here because `update_oneliner` needs to
/// settle them to `OnelinerState::Empty`. The previous implementation
/// required *content* specifically, which left silent-only cases stuck
/// with no oneliner state — and the UI stuck on "Generating".
/// Three terminal shapes count:
/// - `.transcript.txt` with content → [`TranscriptState::Content`]
/// - `.transcript.txt` empty → [`TranscriptState::Silent`]
/// - `.m4a.failed` → permanent transcription failure (ffmpeg remux or
/// Whisper 4xx). Transient Whisper errors leave the file as plain
/// `.m4a` without sidecar (Pending), so they do *not* match here —
/// the page-load heal re-enqueues them on the next refresh.
///
/// Cases where every recording is still `Pending` (plain `.m4a`, no
/// sidecar, not marked failed) return false: the transcriber hasn't
/// finished yet and there is nothing for the oneliner worker to act
/// on.
///
/// Both Silent and permanent-failed count because `update_oneliner`
/// settles each to [`OnelinerState::Empty`] via the same no-content
/// branch in `worker::update_oneliner`.
async fn has_any_transcript(case_dir: &Path) -> bool {
let Ok(mut files) = tokio::fs::read_dir(case_dir).await else {
return false;
@@ -173,7 +183,7 @@ async fn has_any_transcript(case_dir: &Path) -> bool {
while let Ok(Some(f)) = files.next_entry().await {
if f.file_name()
.to_str()
.is_some_and(|s| s.ends_with(".transcript.txt"))
.is_some_and(|s| s.ends_with(".transcript.txt") || s.ends_with(".m4a.failed"))
{
return true;
}
@@ -356,4 +366,42 @@ mod tests {
assert!(cases_needing_oneliner(data.path()).await.is_empty());
}
/// `.m4a.failed`-only cases are terminal — the pipeline will produce
/// no further artefacts. Recovery must pick them up so
/// `update_oneliner` can persist `OnelinerState::Empty`; otherwise
/// the case would rely on the UI fallback forever and never have a
/// stable on-disk state.
#[tokio::test]
async fn cases_needing_oneliner_picks_up_failed_only_cases() {
let data = tempdir().unwrap();
let case = data.path().join("user").join("case1");
tokio::fs::create_dir_all(&case).await.unwrap();
tokio::fs::write(case.join("2026-04-16T10-00-00Z.m4a.failed"), b"x")
.await
.unwrap();
let got = cases_needing_oneliner(data.path()).await;
assert_eq!(got, vec![(case, "user".to_owned())]);
}
#[tokio::test]
async fn has_any_transcript_true_for_failed_only_case() {
let case = tempdir().unwrap();
tokio::fs::write(case.path().join("2026-04-16T10-00-00Z.m4a.failed"), b"x")
.await
.unwrap();
assert!(has_any_transcript(case.path()).await);
}
#[tokio::test]
async fn has_any_transcript_false_for_pending_only_case() {
let case = tempdir().unwrap();
tokio::fs::write(case.path().join("2026-04-16T10-00-00Z.m4a"), b"audio")
.await
.unwrap();
assert!(!has_any_transcript(case.path()).await);
}
}
+74
View File
@@ -27,6 +27,35 @@ impl std::fmt::Display for WhisperError {
impl std::error::Error for WhisperError {}
impl WhisperError {
/// Classify this error as transient (retry will likely succeed later) or
/// permanent (retrying the same input will hit the same wall).
///
/// Rationale: the transcribe worker uses this to decide whether to leave
/// the `.m4a` in place (page-load heal re-enqueues it) or rename it to
/// `.m4a.failed` (terminal; only a manual reset un-fails it).
///
/// - `Http`: reqwest wraps all transport issues (DNS, connect, TLS, read,
/// timeout, mid-flight reset) — every one of these is a candidate for
/// „Minerva is back in a minute", so treat as transient.
/// - `Status`: 5xx is server-side trouble; 408 (Request Timeout) and 429
/// (Too Many Requests) are explicit retry signals. Everything else in
/// the 4xx range (400 Bad Request, 415 Unsupported Media, ...) means
/// the payload itself is the problem and replays will fail identically.
/// - `Io`: local filesystem error while reading the audio file. Either
/// the file is gone or the disk is misbehaving; a retry of the same
/// path is not the right response.
pub fn is_transient(&self) -> bool {
match self {
Self::Http(_) => true,
Self::Status { status, .. } => {
*status == 408 || *status == 429 || (500..=599).contains(status)
}
Self::Io(_) => false,
}
}
}
/// POST the audio file to whisper-asr-webservice and return the transcript text.
///
/// `output=txt` → plain-text response. `language` pins the language so the model
@@ -97,3 +126,48 @@ pub async fn transcribe(
Ok(body)
}
#[cfg(test)]
mod tests {
use super::*;
/// Table-driven classification test. `Http` variant is not publicly
/// constructible (reqwest owns its error ctor), so the transient-Http
/// path is covered by the integration test in
/// `tests/transient_failure_retries_test.rs`. Same code path is exercised
/// here via a 503 Status, which also routes to `is_transient == true`.
#[test]
fn is_transient_classifies_status_codes() {
let cases: &[(u16, bool)] = &[
(400, false),
(401, false),
(403, false),
(404, false),
(408, true),
(415, false),
(422, false),
(429, true),
(500, true),
(502, true),
(503, true),
(504, true),
];
for &(status, expected) in cases {
let err = WhisperError::Status {
status,
body: String::new(),
};
assert_eq!(
err.is_transient(),
expected,
"status {status} expected is_transient={expected}"
);
}
}
#[test]
fn is_transient_io_is_permanent() {
let err = WhisperError::Io(std::io::Error::other("disk gone"));
assert!(!err.is_transient());
}
}
+14 -1
View File
@@ -78,7 +78,20 @@ pub async fn run(
Ok(t) => t,
Err(e) => {
error!(audio = %audio_path.display(), error = %e, "whisper call failed");
mark_failed(&audio_path, &events_tx, &job.user_slug).await;
if e.is_transient() {
// Leave the recording as plain `.m4a` (no `.failed` suffix)
// so the page-load heal (`enqueue_pending_for_user`) picks
// it up on the next refresh. The natural retry trigger is
// a human navigating the UI — no scheduler, no sidecar.
info!(
audio = %audio_path.display(),
error = %e,
reason = "transient",
"recording left as .m4a, page-load heal will re-enqueue"
);
} else {
mark_failed(&audio_path, &events_tx, &job.user_slug).await;
}
continue;
}
};
+6 -3
View File
@@ -67,6 +67,9 @@
.status-badge.done {
background: #7ed321;
}
.status-badge.fehler {
background: #c00;
}
.actions {
margin: 1em 0 1.5em;
display: flex;
@@ -255,9 +258,9 @@
</header>
<h1>
<span class="title">
{% call ol::render(oneliner) %} {% if has_document %}<span
class="status-badge done"
>ausgewertet</span
{% call ol::render(oneliner) %} {% if has_failed_recording
%}<span class="status-badge fehler">fehler</span>{% else if
has_document %}<span class="status-badge done">ausgewertet</span
>{% else %}<span class="status-badge open">offen</span>{% endif
%}
</span>
+1 -1
View File
@@ -114,7 +114,7 @@ try {
</div>
{% if is_admin %}<div class="filename admin-only">{{ rec.filename }}</div>{% endif %}
{% if rec.failed %}
<div class="failed">Transkription fehlgeschlagen — .failed-Suffix entfernen zum Erneut-Versuchen.</div>
<div class="failed">Transkription fehlgeschlagen</div>
{% else %}
{% match rec.transcript %}
{% when TranscriptState::Content with (t) %}
+2 -1
View File
@@ -32,6 +32,7 @@ section h2 .count { font-weight: normal; font-size: 0.85em; color: #888; }
.status-badge { display: inline-block; padding: 0.05em 0.5em; border-radius: 999px; font-size: 0.75em; font-weight: bold; color: white; }
.status-badge.open { background: #e24a4a; }
.status-badge.done { background: #7ed321; }
.status-badge.fehler { background: #c00; }
.case-row .actions { display: flex; gap: 0.4em; align-items: center; }
.case-row .actions button { font-size: 0.85em; padding: 0.35em 0.8em; }
.case-row .actions .delete-btn { padding: 0.3em; background: transparent; border: none; color: #888; cursor: pointer; display: inline-flex; align-items: center; line-height: 0; border-radius: 4px; transition: color 0.15s, background 0.15s; }
@@ -96,7 +97,7 @@ try {
<div class="body">
<a href="/web/cases/{{ case.case_id }}{% if case.is_closed %}?show_closed=1{% endif %}">
<div class="line1"><span class="time"><time datetime="{{ case.recorded_at_iso }}">{{ case.time_hms_utc }}</time></span> — {% call ol::render(case.oneliner) %}</div>
<div class="line2">{{ case.recordings_count }} Aufnahmen{% if case.has_document %} — <span class="status-badge done">ausgewertet</span>{% else if !case.is_closed %} — <span class="status-badge open">offen</span>{% endif %}{% if case.analyzing %} <span class="label analyzing">wird analysiert</span>{% endif %}{% if case.is_closed %}<span class="closed-badge">geschlossen{% match case.days_until_purge %}{% when Some with (d) %} — wird in {{ d }} Tagen entfernt{% when None %}{% endmatch %}</span>{% endif %}</div>
<div class="line2">{{ case.recordings_count }} Aufnahmen{% if case.has_failed_recording %} — <span class="status-badge fehler">fehler</span>{% else if case.has_document %} — <span class="status-badge done">ausgewertet</span>{% else if !case.is_closed %} — <span class="status-badge open">offen</span>{% endif %}{% if case.analyzing %} <span class="label analyzing">wird analysiert</span>{% endif %}{% if case.is_closed %}<span class="closed-badge">geschlossen{% match case.days_until_purge %}{% when Some with (d) %} — wird in {{ d }} Tagen entfernt{% when None %}{% endmatch %}</span>{% endif %}</div>
{% if is_admin %}<div class="uuid admin-only">{{ case.case_id }}</div>{% endif %}
</a>
</div>
@@ -0,0 +1,99 @@
//! Regression test: a case whose recordings are all `.m4a.failed`
//! (permanent transcription failures — ffmpeg corrupt audio or Whisper
//! 4xx) must produce an `OnelinerState::Empty` automatically, so the UI
//! has a durable terminal state instead of relying on the on-render
//! `compute_oneliner_display` fallback.
//!
//! The historical bug path: `.m4a.failed`-only cases were skipped by the
//! recovery scan (`has_any_transcript` looked for `*.transcript.txt`
//! only), so `update_oneliner` never ran, so no `oneliner.json` was
//! written. The UI masked the symptom with its `non_failed_count == 0`
//! fallback — but a future refactor of that fallback would regress the
//! case. The fix makes `has_any_transcript` count `.m4a.failed` too, so
//! `update_oneliner` runs, sees no content transcripts, and settles the
//! case to `Empty` through the same terminal branch used for silent-only
//! cases.
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use doctate_common::oneliners::OnelinerState;
use doctate_server::config::Config;
use doctate_server::events;
use doctate_server::gazetteer::Gazetteer;
use doctate_server::{
AnalyzeBusy, OnelinerHealBusy, PipelineState, TranscribeBusy, WorkerBusy, analyze, transcribe,
};
use tempfile::tempdir;
use wiremock::matchers::{method, path as wm_path};
use wiremock::{Mock, MockServer, ResponseTemplate};
#[tokio::test]
async fn failed_only_case_settles_to_empty_without_llm_call() {
let data = tempdir().unwrap();
let slug = "dr_test";
let user_root = data.path().join(slug);
let case_dir = user_root.join("case-failed");
std::fs::create_dir_all(&case_dir).unwrap();
// Seed: only a `.m4a.failed` file. No transcript, no plain .m4a.
std::fs::write(case_dir.join("2026-04-15T20-30-00Z.m4a.failed"), b"x").unwrap();
// Ollama must NOT be called — nothing to summarize for a failed-only
// case. `.expect(0)` fails the test (on MockServer drop) if any
// request arrived.
let mock = MockServer::start().await;
Mock::given(method("POST"))
.and(wm_path("/api/chat"))
.respond_with(ResponseTemplate::new(200))
.expect(0)
.mount(&mock)
.await;
let mut cfg = Config::test_default();
cfg.data_path = data.path().to_path_buf();
cfg.ollama_url = mock.uri();
let config = Arc::new(cfg);
let vocab = Arc::new(Gazetteer::empty());
let events_tx = events::channel();
let http_client = reqwest::Client::new();
let (tx_a, _rx_a) = analyze::channel();
let (tx_t, _rx_t) = transcribe::channel();
let heal_busy: WorkerBusy = Arc::new(AtomicBool::new(false));
let pipeline = PipelineState {
analyze_busy: AnalyzeBusy(Arc::new(AtomicBool::new(false))),
analyze_tx: tx_a,
transcribe_busy: TranscribeBusy(Arc::new(AtomicBool::new(false))),
transcribe_tx: tx_t,
oneliner_heal_busy: OnelinerHealBusy(heal_busy.clone()),
};
pipeline
.heal_orphans_if_idle(&user_root, slug, &http_client, &config, &vocab, &events_tx)
.await;
// Heal spawn drops the busy flag on completion — poll until idle.
let start = std::time::Instant::now();
while heal_busy.load(Ordering::Acquire) {
if start.elapsed() > Duration::from_secs(5) {
panic!("oneliner heal did not finish within 5s");
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
let path = case_dir.join("oneliner.json");
assert!(
path.exists(),
"expected oneliner.json to be written for failed-only case"
);
let bytes = std::fs::read(&path).unwrap();
let state: OnelinerState = serde_json::from_slice(&bytes).unwrap();
assert!(
matches!(state, OnelinerState::Empty { .. }),
"expected OnelinerState::Empty, got {state:?}"
);
// Mock.drop() runs here — `.expect(0)` panics if Ollama was touched.
}
+62 -2
View File
@@ -276,12 +276,17 @@ fn test_config_with_whisper(whisper_url: String) -> Arc<Config> {
})
}
/// Permanent Whisper errors (4xx other than 408/429) must rename `.m4a` to
/// `.m4a.failed` so recovery treats the recording as terminal. HTTP 400 is
/// the archetypal "the payload itself is wrong" response — retrying the
/// same bytes will hit the same wall, so the file is marked and only a
/// manual admin reset un-fails it.
#[tokio::test]
async fn worker_renames_audio_to_failed_on_whisper_error() {
async fn worker_renames_audio_to_failed_on_permanent_whisper_error() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/asr"))
.respond_with(ResponseTemplate::new(500).set_body_string("boom"))
.respond_with(ResponseTemplate::new(400).set_body_string("bad payload"))
.mount(&server)
.await;
@@ -321,6 +326,61 @@ async fn worker_renames_audio_to_failed_on_whisper_error() {
assert!(failed.exists(), "expected {} to exist", failed.display());
}
/// Transient Whisper errors (5xx / 408 / 429 / network) must NOT rename
/// the audio. The file stays as plain `.m4a` so the page-load heal
/// (`enqueue_pending_for_user`) picks it up automatically on the next
/// refresh — which is what happens when Minerva recovers.
#[tokio::test]
async fn worker_leaves_m4a_intact_on_transient_whisper_error() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/asr"))
.respond_with(ResponseTemplate::new(503).set_body_string("service unavailable"))
.mount(&server)
.await;
let tmp = tempfile::tempdir().unwrap();
let case_dir = tmp.path().join("dr_test/bbbb");
std::fs::create_dir_all(&case_dir).unwrap();
let audio = case_dir.join("2026-04-13T10-31-00Z.m4a");
std::fs::copy(fixture("sample.m4a"), &audio).unwrap();
let config = test_config_with_whisper(server.uri());
let (tx, rx) = transcribe::channel();
tx.send(transcribe::TranscribeJob {
audio_path: audio.clone(),
user_slug: "dr_test".into(),
})
.await
.unwrap();
drop(tx);
let client = reqwest::Client::new();
let busy = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let vocab = std::sync::Arc::new(doctate_server::gazetteer::Gazetteer::empty());
transcribe::worker::run(
rx,
config,
client,
busy,
vocab,
doctate_server::events::channel(),
)
.await;
// Original .m4a must still be there; .failed must NOT exist.
assert!(audio.exists(), "transient error must not consume the .m4a");
let failed = case_dir.join("2026-04-13T10-31-00Z.m4a.failed");
assert!(
!failed.exists(),
"transient error must not mark {} as failed",
failed.display()
);
// And no transcript yet either (the 503 produced no text).
let transcript = case_dir.join("2026-04-13T10-31-00Z.transcript.txt");
assert!(!transcript.exists(), "no transcript expected on 503");
}
/// The worker must route the Whisper response through the Gazetteer
/// before persisting. Mock returns the drift form "Zerebrum"; the
/// persisted `.transcript.txt` must contain the canonical "Cerebrum".
@@ -0,0 +1,172 @@
//! End-to-end regression test for the transient-transcribe-failure heal.
//!
//! Scenario: Minerva is briefly offline (HTTP 503) when the doctor's
//! recording arrives. The transcribe worker classifies the error as
//! transient and leaves the audio as plain `.m4a` — no `.failed`
//! suffix. On the next page-load, `heal_orphans_if_idle` re-enqueues
//! the pending `.m4a`. By then Minerva is back; Whisper returns the
//! transcript and the case advances normally.
//!
//! Pre-fix behaviour: any Whisper error renamed `.m4a` → `.m4a.failed`,
//! terminating progress permanently. Only a manual admin reset un-failed
//! the recording.
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use doctate_server::config::{Config, User};
use doctate_server::events;
use doctate_server::gazetteer::Gazetteer;
use doctate_server::{
AnalyzeBusy, OnelinerHealBusy, PipelineState, TranscribeBusy, WorkerBusy, analyze, transcribe,
};
use tempfile::tempdir;
use wiremock::matchers::{method, path as wm_path};
use wiremock::{Mock, MockServer, ResponseTemplate};
fn fixture(name: &str) -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures")
.join(name)
}
#[tokio::test]
async fn transient_whisper_failure_is_reenqueued_and_recovers() {
// Arrange filesystem: one real m4a for user "dr_test".
let data = tempdir().unwrap();
let slug = "dr_test";
let user_root = data.path().join(slug);
let case_dir = user_root.join("case-transient");
std::fs::create_dir_all(&case_dir).unwrap();
let audio = case_dir.join("2026-04-15T20-30-00Z.m4a");
std::fs::copy(fixture("sample.m4a"), &audio).unwrap();
// Arrange Whisper mock: first call → 503 (Minerva down),
// subsequent calls → 200 with a transcript. wiremock matches
// mocks in registration order; `up_to_n_times(1)` retires the
// first mock after one hit, so the second takes over for any
// retry.
let whisper = MockServer::start().await;
Mock::given(method("POST"))
.and(wm_path("/asr"))
.respond_with(ResponseTemplate::new(503).set_body_string("service unavailable"))
.up_to_n_times(1)
.mount(&whisper)
.await;
Mock::given(method("POST"))
.and(wm_path("/asr"))
.respond_with(ResponseTemplate::new(200).set_body_string("recovered text"))
.mount(&whisper)
.await;
// Config points at the mock Whisper, ollama intentionally unreachable
// (the oneliner heal may fire in parallel; we only assert on
// the transcript artefact, not on the oneliner).
let mut cfg = Config::test_default();
cfg.data_path = data.path().to_path_buf();
cfg.whisper_url = whisper.uri();
cfg.whisper_timeout_seconds = 5;
cfg.users = vec![User {
slug: slug.into(),
api_key: "k".into(),
web_password: "unused".into(),
role: "doctor".into(),
whisper: Default::default(),
retention: Default::default(),
}];
let config = Arc::new(cfg);
let vocab = Arc::new(Gazetteer::empty());
let events_tx = events::channel();
let http_client = reqwest::Client::new();
let (tx_a, _rx_a) = analyze::channel();
let (tx_t, rx_t) = transcribe::channel();
let transcribe_busy: WorkerBusy = Arc::new(AtomicBool::new(false));
let heal_busy: WorkerBusy = Arc::new(AtomicBool::new(false));
// Spawn the transcribe worker in the background; it processes jobs
// until the channel is closed at test teardown.
let worker = tokio::spawn(transcribe::worker::run(
rx_t,
config.clone(),
http_client.clone(),
transcribe_busy.clone(),
vocab.clone(),
events_tx.clone(),
));
let pipeline = PipelineState {
analyze_busy: AnalyzeBusy(Arc::new(AtomicBool::new(false))),
analyze_tx: tx_a,
transcribe_busy: TranscribeBusy(transcribe_busy.clone()),
transcribe_tx: tx_t.clone(),
oneliner_heal_busy: OnelinerHealBusy(heal_busy.clone()),
};
// Act 1: enqueue the initial job, wait for the 503 failure.
tx_t.send(transcribe::TranscribeJob {
audio_path: audio.clone(),
user_slug: slug.into(),
})
.await
.unwrap();
// Wait until the worker has processed the first job (busy false
// AND the 503 mock has recorded a hit). Polling both conditions
// avoids a race where the worker hasn't started yet.
let start = std::time::Instant::now();
loop {
let idle = !transcribe_busy.load(Ordering::Acquire);
let hits = whisper.received_requests().await.unwrap().len();
if idle && hits >= 1 {
break;
}
if start.elapsed() > Duration::from_secs(10) {
panic!("worker did not process first job within 10s (idle={idle}, hits={hits})");
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
// Invariant after transient failure: .m4a still there, no .failed.
assert!(audio.exists(), "transient error must not consume the .m4a");
assert!(
!case_dir.join("2026-04-15T20-30-00Z.m4a.failed").exists(),
".m4a.failed must not exist after transient error"
);
assert!(
!case_dir
.join("2026-04-15T20-30-00Z.transcript.txt")
.exists(),
"no transcript yet — the 503 produced nothing"
);
// Act 2: heal re-enqueues the pending .m4a (second mock → 200).
pipeline
.heal_orphans_if_idle(&user_root, slug, &http_client, &config, &vocab, &events_tx)
.await;
// Assert: the transcript lands. Poll because the worker runs
// asynchronously after the heal hands off the job.
let transcript = case_dir.join("2026-04-15T20-30-00Z.transcript.txt");
let start = std::time::Instant::now();
while !transcript.exists() {
if start.elapsed() > Duration::from_secs(10) {
panic!(
"transcript did not appear within 10s (whisper hits: {})",
whisper.received_requests().await.unwrap().len()
);
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
let body = std::fs::read_to_string(&transcript).unwrap();
assert_eq!(body, "recovered text");
// Teardown: close the worker channel so the background task exits.
drop(tx_t);
drop(pipeline);
let _ = tokio::time::timeout(Duration::from_secs(5), worker).await;
}