fix: separate transient from permanent transcribe failures
Before, every Whisper error (5xx, timeout, Minerva down, corrupt audio,
4xx) renamed `<ts>.m4a` to `<ts>.m4a.failed` uniformly, turning transient
outages into permanent sackgassen that only a manual admin reset could
undo. Cases whose every recording was `.m4a.failed` stuck without a
persisted oneliner state; the UI masked them via a fallback in
`compute_oneliner_display`.
The core insight: a recoverable error is not an error. Transient Whisper
failures now leave the audio as plain `.m4a` so the existing page-load
heal (`enqueue_pending_for_user`) re-enqueues it on the next refresh —
which is exactly what happens when Minerva comes back. No new sidecar,
no retry count, no scheduler.
Changes:
- `WhisperError::is_transient` classifies `Http`, 5xx, 408, 429 as
transient; `Io` and other 4xx as permanent.
- Transcribe worker: transient → info log + continue (audio stays .m4a);
permanent → `mark_failed` as before.
- `has_any_transcript` counts `.m4a.failed` as terminal, so
`update_oneliner` runs for failed-only cases and settles
`OnelinerState::Empty` (analogous to the silent-only fix in 4531f85).
- New verdrängender `fehler`-Badge (#c00) in case list and detail when
at least one `.m4a.failed` exists; recording-level message shortened
to plain "Transkription fehlgeschlagen".
Tests:
- New integration: transient 503 leaves `.m4a` intact, heal recovers.
- New integration: `.m4a.failed`-only case settles to
`OnelinerState::Empty` without calling Ollama.
- New unit: `is_transient` table test across relevant status codes.
- New unit: `has_any_transcript` returns true for `.m4a.failed`-only
case and false for pending-only case.
- Existing worker test retargeted from 500 to 400 and renamed; added
companion `worker_leaves_m4a_intact_on_transient_whisper_error`.
This commit is contained in:
@@ -71,6 +71,12 @@ struct UserCaseView {
|
||||
recordings_count: usize,
|
||||
analyzing: bool,
|
||||
has_document: bool,
|
||||
/// True iff at least one recording is `.m4a.failed` — a *permanent*
|
||||
/// transcribe failure (ffmpeg remux or Whisper 4xx). Transient Whisper
|
||||
/// errors never set this flag because they leave the file as plain
|
||||
/// `.m4a` for the page-load heal to retry. Drives the verdrängende
|
||||
/// `fehler`-Badge over `offen`/`ausgewertet`.
|
||||
has_failed_recording: bool,
|
||||
/// True iff the inline "Analysieren"-button should be enabled.
|
||||
/// Requires: not currently analyzing, all non-failed recordings
|
||||
/// transcribed, and an LLM is configured.
|
||||
@@ -213,6 +219,10 @@ struct CasePageTemplate {
|
||||
llm_missing: bool,
|
||||
analyzing: bool,
|
||||
has_document: bool,
|
||||
/// True iff at least one `.m4a.failed` recording exists. Drives the
|
||||
/// verdrängende `fehler`-Badge in the case header; see the same
|
||||
/// field on `UserCaseView` for semantics.
|
||||
has_failed_recording: bool,
|
||||
is_admin: bool,
|
||||
/// True when the case carries a `.closed` marker. Toggles the
|
||||
/// header action button between close and reopen, and appends
|
||||
@@ -491,6 +501,7 @@ pub async fn handle_case_page(
|
||||
.iter()
|
||||
.filter(|r| r.transcript.has_file())
|
||||
.count();
|
||||
let has_failed_recording = recordings.iter().any(|r| r.failed);
|
||||
let case_id_short = case_id_str.chars().take(8).collect();
|
||||
let is_admin = user.is_admin();
|
||||
|
||||
@@ -515,6 +526,7 @@ pub async fn handle_case_page(
|
||||
llm_missing: flags.llm_missing,
|
||||
analyzing: flags.analyzing,
|
||||
has_document: flags.has_document,
|
||||
has_failed_recording,
|
||||
is_admin,
|
||||
is_closed,
|
||||
}
|
||||
@@ -732,6 +744,7 @@ async fn compute_case_view(
|
||||
.unwrap_or_default();
|
||||
let recordings_count = recordings.len();
|
||||
let non_failed_count = recordings.iter().filter(|r| !r.failed).count();
|
||||
let has_failed_recording = recordings.iter().any(|r| r.failed);
|
||||
let all_transcribed = non_failed_count > 0
|
||||
&& recordings
|
||||
.iter()
|
||||
@@ -757,6 +770,7 @@ async fn compute_case_view(
|
||||
recordings_count,
|
||||
analyzing,
|
||||
has_document,
|
||||
has_failed_recording,
|
||||
can_analyze,
|
||||
is_closed,
|
||||
days_until_purge,
|
||||
|
||||
@@ -156,16 +156,26 @@ pub(crate) async fn cases_needing_oneliner_in(user_root: &Path) -> Vec<PathBuf>
|
||||
out
|
||||
}
|
||||
|
||||
/// True iff at least one `.transcript.txt` sidecar exists in `case_dir`
|
||||
/// — i.e., at least one recording is either [`TranscriptState::Silent`]
|
||||
/// or [`TranscriptState::Content`]. Cases where every recording is
|
||||
/// still `Pending` return false: there's nothing for the oneliner
|
||||
/// worker to act on yet.
|
||||
/// True iff the case has at least one recording in a *terminal* state —
|
||||
/// meaning the transcribe pipeline will produce no further artefacts
|
||||
/// and `update_oneliner` has enough information to settle the case.
|
||||
///
|
||||
/// Silent transcripts count here because `update_oneliner` needs to
|
||||
/// settle them to `OnelinerState::Empty`. The previous implementation
|
||||
/// required *content* specifically, which left silent-only cases stuck
|
||||
/// with no oneliner state — and the UI stuck on "Generating".
|
||||
/// Three terminal shapes count:
|
||||
/// - `.transcript.txt` with content → [`TranscriptState::Content`]
|
||||
/// - `.transcript.txt` empty → [`TranscriptState::Silent`]
|
||||
/// - `.m4a.failed` → permanent transcription failure (ffmpeg remux or
|
||||
/// Whisper 4xx). Transient Whisper errors leave the file as plain
|
||||
/// `.m4a` without sidecar (Pending), so they do *not* match here —
|
||||
/// the page-load heal re-enqueues them on the next refresh.
|
||||
///
|
||||
/// Cases where every recording is still `Pending` (plain `.m4a`, no
|
||||
/// sidecar, not marked failed) return false: the transcriber hasn't
|
||||
/// finished yet and there is nothing for the oneliner worker to act
|
||||
/// on.
|
||||
///
|
||||
/// Both Silent and permanent-failed count because `update_oneliner`
|
||||
/// settles each to [`OnelinerState::Empty`] via the same no-content
|
||||
/// branch in `worker::update_oneliner`.
|
||||
async fn has_any_transcript(case_dir: &Path) -> bool {
|
||||
let Ok(mut files) = tokio::fs::read_dir(case_dir).await else {
|
||||
return false;
|
||||
@@ -173,7 +183,7 @@ async fn has_any_transcript(case_dir: &Path) -> bool {
|
||||
while let Ok(Some(f)) = files.next_entry().await {
|
||||
if f.file_name()
|
||||
.to_str()
|
||||
.is_some_and(|s| s.ends_with(".transcript.txt"))
|
||||
.is_some_and(|s| s.ends_with(".transcript.txt") || s.ends_with(".m4a.failed"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
@@ -356,4 +366,42 @@ mod tests {
|
||||
|
||||
assert!(cases_needing_oneliner(data.path()).await.is_empty());
|
||||
}
|
||||
|
||||
/// `.m4a.failed`-only cases are terminal — the pipeline will produce
|
||||
/// no further artefacts. Recovery must pick them up so
|
||||
/// `update_oneliner` can persist `OnelinerState::Empty`; otherwise
|
||||
/// the case would rely on the UI fallback forever and never have a
|
||||
/// stable on-disk state.
|
||||
#[tokio::test]
|
||||
async fn cases_needing_oneliner_picks_up_failed_only_cases() {
|
||||
let data = tempdir().unwrap();
|
||||
let case = data.path().join("user").join("case1");
|
||||
tokio::fs::create_dir_all(&case).await.unwrap();
|
||||
tokio::fs::write(case.join("2026-04-16T10-00-00Z.m4a.failed"), b"x")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let got = cases_needing_oneliner(data.path()).await;
|
||||
assert_eq!(got, vec![(case, "user".to_owned())]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn has_any_transcript_true_for_failed_only_case() {
|
||||
let case = tempdir().unwrap();
|
||||
tokio::fs::write(case.path().join("2026-04-16T10-00-00Z.m4a.failed"), b"x")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(has_any_transcript(case.path()).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn has_any_transcript_false_for_pending_only_case() {
|
||||
let case = tempdir().unwrap();
|
||||
tokio::fs::write(case.path().join("2026-04-16T10-00-00Z.m4a"), b"audio")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!has_any_transcript(case.path()).await);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,35 @@ impl std::fmt::Display for WhisperError {
|
||||
|
||||
impl std::error::Error for WhisperError {}
|
||||
|
||||
impl WhisperError {
|
||||
/// Classify this error as transient (retry will likely succeed later) or
|
||||
/// permanent (retrying the same input will hit the same wall).
|
||||
///
|
||||
/// Rationale: the transcribe worker uses this to decide whether to leave
|
||||
/// the `.m4a` in place (page-load heal re-enqueues it) or rename it to
|
||||
/// `.m4a.failed` (terminal; only a manual reset un-fails it).
|
||||
///
|
||||
/// - `Http`: reqwest wraps all transport issues (DNS, connect, TLS, read,
|
||||
/// timeout, mid-flight reset) — every one of these is a candidate for
|
||||
/// „Minerva is back in a minute", so treat as transient.
|
||||
/// - `Status`: 5xx is server-side trouble; 408 (Request Timeout) and 429
|
||||
/// (Too Many Requests) are explicit retry signals. Everything else in
|
||||
/// the 4xx range (400 Bad Request, 415 Unsupported Media, ...) means
|
||||
/// the payload itself is the problem and replays will fail identically.
|
||||
/// - `Io`: local filesystem error while reading the audio file. Either
|
||||
/// the file is gone or the disk is misbehaving; a retry of the same
|
||||
/// path is not the right response.
|
||||
pub fn is_transient(&self) -> bool {
|
||||
match self {
|
||||
Self::Http(_) => true,
|
||||
Self::Status { status, .. } => {
|
||||
*status == 408 || *status == 429 || (500..=599).contains(status)
|
||||
}
|
||||
Self::Io(_) => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// POST the audio file to whisper-asr-webservice and return the transcript text.
|
||||
///
|
||||
/// `output=txt` → plain-text response. `language` pins the language so the model
|
||||
@@ -97,3 +126,48 @@ pub async fn transcribe(
|
||||
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Table-driven classification test. `Http` variant is not publicly
|
||||
/// constructible (reqwest owns its error ctor), so the transient-Http
|
||||
/// path is covered by the integration test in
|
||||
/// `tests/transient_failure_retries_test.rs`. Same code path is exercised
|
||||
/// here via a 503 Status, which also routes to `is_transient == true`.
|
||||
#[test]
|
||||
fn is_transient_classifies_status_codes() {
|
||||
let cases: &[(u16, bool)] = &[
|
||||
(400, false),
|
||||
(401, false),
|
||||
(403, false),
|
||||
(404, false),
|
||||
(408, true),
|
||||
(415, false),
|
||||
(422, false),
|
||||
(429, true),
|
||||
(500, true),
|
||||
(502, true),
|
||||
(503, true),
|
||||
(504, true),
|
||||
];
|
||||
for &(status, expected) in cases {
|
||||
let err = WhisperError::Status {
|
||||
status,
|
||||
body: String::new(),
|
||||
};
|
||||
assert_eq!(
|
||||
err.is_transient(),
|
||||
expected,
|
||||
"status {status} expected is_transient={expected}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_transient_io_is_permanent() {
|
||||
let err = WhisperError::Io(std::io::Error::other("disk gone"));
|
||||
assert!(!err.is_transient());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,7 +78,20 @@ pub async fn run(
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
error!(audio = %audio_path.display(), error = %e, "whisper call failed");
|
||||
mark_failed(&audio_path, &events_tx, &job.user_slug).await;
|
||||
if e.is_transient() {
|
||||
// Leave the recording as plain `.m4a` (no `.failed` suffix)
|
||||
// so the page-load heal (`enqueue_pending_for_user`) picks
|
||||
// it up on the next refresh. The natural retry trigger is
|
||||
// a human navigating the UI — no scheduler, no sidecar.
|
||||
info!(
|
||||
audio = %audio_path.display(),
|
||||
error = %e,
|
||||
reason = "transient",
|
||||
"recording left as .m4a, page-load heal will re-enqueue"
|
||||
);
|
||||
} else {
|
||||
mark_failed(&audio_path, &events_tx, &job.user_slug).await;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -67,6 +67,9 @@
|
||||
.status-badge.done {
|
||||
background: #7ed321;
|
||||
}
|
||||
.status-badge.fehler {
|
||||
background: #c00;
|
||||
}
|
||||
.actions {
|
||||
margin: 1em 0 1.5em;
|
||||
display: flex;
|
||||
@@ -255,9 +258,9 @@
|
||||
</header>
|
||||
<h1>
|
||||
<span class="title">
|
||||
{% call ol::render(oneliner) %} {% if has_document %}<span
|
||||
class="status-badge done"
|
||||
>ausgewertet</span
|
||||
{% call ol::render(oneliner) %} {% if has_failed_recording
|
||||
%}<span class="status-badge fehler">fehler</span>{% else if
|
||||
has_document %}<span class="status-badge done">ausgewertet</span
|
||||
>{% else %}<span class="status-badge open">offen</span>{% endif
|
||||
%}
|
||||
</span>
|
||||
|
||||
@@ -114,7 +114,7 @@ try {
|
||||
</div>
|
||||
{% if is_admin %}<div class="filename admin-only">{{ rec.filename }}</div>{% endif %}
|
||||
{% if rec.failed %}
|
||||
<div class="failed">Transkription fehlgeschlagen — .failed-Suffix entfernen zum Erneut-Versuchen.</div>
|
||||
<div class="failed">Transkription fehlgeschlagen</div>
|
||||
{% else %}
|
||||
{% match rec.transcript %}
|
||||
{% when TranscriptState::Content with (t) %}
|
||||
|
||||
@@ -32,6 +32,7 @@ section h2 .count { font-weight: normal; font-size: 0.85em; color: #888; }
|
||||
.status-badge { display: inline-block; padding: 0.05em 0.5em; border-radius: 999px; font-size: 0.75em; font-weight: bold; color: white; }
|
||||
.status-badge.open { background: #e24a4a; }
|
||||
.status-badge.done { background: #7ed321; }
|
||||
.status-badge.fehler { background: #c00; }
|
||||
.case-row .actions { display: flex; gap: 0.4em; align-items: center; }
|
||||
.case-row .actions button { font-size: 0.85em; padding: 0.35em 0.8em; }
|
||||
.case-row .actions .delete-btn { padding: 0.3em; background: transparent; border: none; color: #888; cursor: pointer; display: inline-flex; align-items: center; line-height: 0; border-radius: 4px; transition: color 0.15s, background 0.15s; }
|
||||
@@ -96,7 +97,7 @@ try {
|
||||
<div class="body">
|
||||
<a href="/web/cases/{{ case.case_id }}{% if case.is_closed %}?show_closed=1{% endif %}">
|
||||
<div class="line1"><span class="time"><time datetime="{{ case.recorded_at_iso }}">{{ case.time_hms_utc }}</time></span> — {% call ol::render(case.oneliner) %}</div>
|
||||
<div class="line2">{{ case.recordings_count }} Aufnahmen{% if case.has_document %} — <span class="status-badge done">ausgewertet</span>{% else if !case.is_closed %} — <span class="status-badge open">offen</span>{% endif %}{% if case.analyzing %} <span class="label analyzing">wird analysiert</span>{% endif %}{% if case.is_closed %}<span class="closed-badge">geschlossen{% match case.days_until_purge %}{% when Some with (d) %} — wird in {{ d }} Tagen entfernt{% when None %}{% endmatch %}</span>{% endif %}</div>
|
||||
<div class="line2">{{ case.recordings_count }} Aufnahmen{% if case.has_failed_recording %} — <span class="status-badge fehler">fehler</span>{% else if case.has_document %} — <span class="status-badge done">ausgewertet</span>{% else if !case.is_closed %} — <span class="status-badge open">offen</span>{% endif %}{% if case.analyzing %} <span class="label analyzing">wird analysiert</span>{% endif %}{% if case.is_closed %}<span class="closed-badge">geschlossen{% match case.days_until_purge %}{% when Some with (d) %} — wird in {{ d }} Tagen entfernt{% when None %}{% endmatch %}</span>{% endif %}</div>
|
||||
{% if is_admin %}<div class="uuid admin-only">{{ case.case_id }}</div>{% endif %}
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
//! Regression test: a case whose recordings are all `.m4a.failed`
|
||||
//! (permanent transcription failures — ffmpeg corrupt audio or Whisper
|
||||
//! 4xx) must produce an `OnelinerState::Empty` automatically, so the UI
|
||||
//! has a durable terminal state instead of relying on the on-render
|
||||
//! `compute_oneliner_display` fallback.
|
||||
//!
|
||||
//! The historical bug path: `.m4a.failed`-only cases were skipped by the
|
||||
//! recovery scan (`has_any_transcript` looked for `*.transcript.txt`
|
||||
//! only), so `update_oneliner` never ran, so no `oneliner.json` was
|
||||
//! written. The UI masked the symptom with its `non_failed_count == 0`
|
||||
//! fallback — but a future refactor of that fallback would regress the
|
||||
//! case. The fix makes `has_any_transcript` count `.m4a.failed` too, so
|
||||
//! `update_oneliner` runs, sees no content transcripts, and settles the
|
||||
//! case to `Empty` through the same terminal branch used for silent-only
|
||||
//! cases.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use doctate_common::oneliners::OnelinerState;
|
||||
use doctate_server::config::Config;
|
||||
use doctate_server::events;
|
||||
use doctate_server::gazetteer::Gazetteer;
|
||||
use doctate_server::{
|
||||
AnalyzeBusy, OnelinerHealBusy, PipelineState, TranscribeBusy, WorkerBusy, analyze, transcribe,
|
||||
};
|
||||
use tempfile::tempdir;
|
||||
use wiremock::matchers::{method, path as wm_path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_only_case_settles_to_empty_without_llm_call() {
|
||||
let data = tempdir().unwrap();
|
||||
let slug = "dr_test";
|
||||
let user_root = data.path().join(slug);
|
||||
let case_dir = user_root.join("case-failed");
|
||||
std::fs::create_dir_all(&case_dir).unwrap();
|
||||
// Seed: only a `.m4a.failed` file. No transcript, no plain .m4a.
|
||||
std::fs::write(case_dir.join("2026-04-15T20-30-00Z.m4a.failed"), b"x").unwrap();
|
||||
|
||||
// Ollama must NOT be called — nothing to summarize for a failed-only
|
||||
// case. `.expect(0)` fails the test (on MockServer drop) if any
|
||||
// request arrived.
|
||||
let mock = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(wm_path("/api/chat"))
|
||||
.respond_with(ResponseTemplate::new(200))
|
||||
.expect(0)
|
||||
.mount(&mock)
|
||||
.await;
|
||||
|
||||
let mut cfg = Config::test_default();
|
||||
cfg.data_path = data.path().to_path_buf();
|
||||
cfg.ollama_url = mock.uri();
|
||||
let config = Arc::new(cfg);
|
||||
|
||||
let vocab = Arc::new(Gazetteer::empty());
|
||||
let events_tx = events::channel();
|
||||
let http_client = reqwest::Client::new();
|
||||
let (tx_a, _rx_a) = analyze::channel();
|
||||
let (tx_t, _rx_t) = transcribe::channel();
|
||||
|
||||
let heal_busy: WorkerBusy = Arc::new(AtomicBool::new(false));
|
||||
let pipeline = PipelineState {
|
||||
analyze_busy: AnalyzeBusy(Arc::new(AtomicBool::new(false))),
|
||||
analyze_tx: tx_a,
|
||||
transcribe_busy: TranscribeBusy(Arc::new(AtomicBool::new(false))),
|
||||
transcribe_tx: tx_t,
|
||||
oneliner_heal_busy: OnelinerHealBusy(heal_busy.clone()),
|
||||
};
|
||||
|
||||
pipeline
|
||||
.heal_orphans_if_idle(&user_root, slug, &http_client, &config, &vocab, &events_tx)
|
||||
.await;
|
||||
|
||||
// Heal spawn drops the busy flag on completion — poll until idle.
|
||||
let start = std::time::Instant::now();
|
||||
while heal_busy.load(Ordering::Acquire) {
|
||||
if start.elapsed() > Duration::from_secs(5) {
|
||||
panic!("oneliner heal did not finish within 5s");
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
|
||||
let path = case_dir.join("oneliner.json");
|
||||
assert!(
|
||||
path.exists(),
|
||||
"expected oneliner.json to be written for failed-only case"
|
||||
);
|
||||
let bytes = std::fs::read(&path).unwrap();
|
||||
let state: OnelinerState = serde_json::from_slice(&bytes).unwrap();
|
||||
assert!(
|
||||
matches!(state, OnelinerState::Empty { .. }),
|
||||
"expected OnelinerState::Empty, got {state:?}"
|
||||
);
|
||||
|
||||
// Mock.drop() runs here — `.expect(0)` panics if Ollama was touched.
|
||||
}
|
||||
@@ -276,12 +276,17 @@ fn test_config_with_whisper(whisper_url: String) -> Arc<Config> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Permanent Whisper errors (4xx other than 408/429) must rename `.m4a` to
|
||||
/// `.m4a.failed` so recovery treats the recording as terminal. HTTP 400 is
|
||||
/// the archetypal "the payload itself is wrong" response — retrying the
|
||||
/// same bytes will hit the same wall, so the file is marked and only a
|
||||
/// manual admin reset un-fails it.
|
||||
#[tokio::test]
|
||||
async fn worker_renames_audio_to_failed_on_whisper_error() {
|
||||
async fn worker_renames_audio_to_failed_on_permanent_whisper_error() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/asr"))
|
||||
.respond_with(ResponseTemplate::new(500).set_body_string("boom"))
|
||||
.respond_with(ResponseTemplate::new(400).set_body_string("bad payload"))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
@@ -321,6 +326,61 @@ async fn worker_renames_audio_to_failed_on_whisper_error() {
|
||||
assert!(failed.exists(), "expected {} to exist", failed.display());
|
||||
}
|
||||
|
||||
/// Transient Whisper errors (5xx / 408 / 429 / network) must NOT rename
|
||||
/// the audio. The file stays as plain `.m4a` so the page-load heal
|
||||
/// (`enqueue_pending_for_user`) picks it up automatically on the next
|
||||
/// refresh — which is what happens when Minerva recovers.
|
||||
#[tokio::test]
|
||||
async fn worker_leaves_m4a_intact_on_transient_whisper_error() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/asr"))
|
||||
.respond_with(ResponseTemplate::new(503).set_body_string("service unavailable"))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let case_dir = tmp.path().join("dr_test/bbbb");
|
||||
std::fs::create_dir_all(&case_dir).unwrap();
|
||||
let audio = case_dir.join("2026-04-13T10-31-00Z.m4a");
|
||||
std::fs::copy(fixture("sample.m4a"), &audio).unwrap();
|
||||
|
||||
let config = test_config_with_whisper(server.uri());
|
||||
let (tx, rx) = transcribe::channel();
|
||||
tx.send(transcribe::TranscribeJob {
|
||||
audio_path: audio.clone(),
|
||||
user_slug: "dr_test".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
drop(tx);
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let busy = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let vocab = std::sync::Arc::new(doctate_server::gazetteer::Gazetteer::empty());
|
||||
transcribe::worker::run(
|
||||
rx,
|
||||
config,
|
||||
client,
|
||||
busy,
|
||||
vocab,
|
||||
doctate_server::events::channel(),
|
||||
)
|
||||
.await;
|
||||
|
||||
// Original .m4a must still be there; .failed must NOT exist.
|
||||
assert!(audio.exists(), "transient error must not consume the .m4a");
|
||||
let failed = case_dir.join("2026-04-13T10-31-00Z.m4a.failed");
|
||||
assert!(
|
||||
!failed.exists(),
|
||||
"transient error must not mark {} as failed",
|
||||
failed.display()
|
||||
);
|
||||
// And no transcript yet either (the 503 produced no text).
|
||||
let transcript = case_dir.join("2026-04-13T10-31-00Z.transcript.txt");
|
||||
assert!(!transcript.exists(), "no transcript expected on 503");
|
||||
}
|
||||
|
||||
/// The worker must route the Whisper response through the Gazetteer
|
||||
/// before persisting. Mock returns the drift form "Zerebrum"; the
|
||||
/// persisted `.transcript.txt` must contain the canonical "Cerebrum".
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
//! End-to-end regression test for the transient-transcribe-failure heal.
|
||||
//!
|
||||
//! Scenario: Minerva is briefly offline (HTTP 503) when the doctor's
|
||||
//! recording arrives. The transcribe worker classifies the error as
|
||||
//! transient and leaves the audio as plain `.m4a` — no `.failed`
|
||||
//! suffix. On the next page-load, `heal_orphans_if_idle` re-enqueues
|
||||
//! the pending `.m4a`. By then Minerva is back; Whisper returns the
|
||||
//! transcript and the case advances normally.
|
||||
//!
|
||||
//! Pre-fix behaviour: any Whisper error renamed `.m4a` → `.m4a.failed`,
|
||||
//! terminating progress permanently. Only a manual admin reset un-failed
|
||||
//! the recording.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use doctate_server::config::{Config, User};
|
||||
use doctate_server::events;
|
||||
use doctate_server::gazetteer::Gazetteer;
|
||||
use doctate_server::{
|
||||
AnalyzeBusy, OnelinerHealBusy, PipelineState, TranscribeBusy, WorkerBusy, analyze, transcribe,
|
||||
};
|
||||
use tempfile::tempdir;
|
||||
use wiremock::matchers::{method, path as wm_path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
fn fixture(name: &str) -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures")
|
||||
.join(name)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transient_whisper_failure_is_reenqueued_and_recovers() {
|
||||
// Arrange filesystem: one real m4a for user "dr_test".
|
||||
let data = tempdir().unwrap();
|
||||
let slug = "dr_test";
|
||||
let user_root = data.path().join(slug);
|
||||
let case_dir = user_root.join("case-transient");
|
||||
std::fs::create_dir_all(&case_dir).unwrap();
|
||||
let audio = case_dir.join("2026-04-15T20-30-00Z.m4a");
|
||||
std::fs::copy(fixture("sample.m4a"), &audio).unwrap();
|
||||
|
||||
// Arrange Whisper mock: first call → 503 (Minerva down),
|
||||
// subsequent calls → 200 with a transcript. wiremock matches
|
||||
// mocks in registration order; `up_to_n_times(1)` retires the
|
||||
// first mock after one hit, so the second takes over for any
|
||||
// retry.
|
||||
let whisper = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(wm_path("/asr"))
|
||||
.respond_with(ResponseTemplate::new(503).set_body_string("service unavailable"))
|
||||
.up_to_n_times(1)
|
||||
.mount(&whisper)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(wm_path("/asr"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_string("recovered text"))
|
||||
.mount(&whisper)
|
||||
.await;
|
||||
|
||||
// Config points at the mock Whisper, ollama intentionally unreachable
|
||||
// (the oneliner heal may fire in parallel; we only assert on
|
||||
// the transcript artefact, not on the oneliner).
|
||||
let mut cfg = Config::test_default();
|
||||
cfg.data_path = data.path().to_path_buf();
|
||||
cfg.whisper_url = whisper.uri();
|
||||
cfg.whisper_timeout_seconds = 5;
|
||||
cfg.users = vec![User {
|
||||
slug: slug.into(),
|
||||
api_key: "k".into(),
|
||||
web_password: "unused".into(),
|
||||
role: "doctor".into(),
|
||||
whisper: Default::default(),
|
||||
retention: Default::default(),
|
||||
}];
|
||||
let config = Arc::new(cfg);
|
||||
|
||||
let vocab = Arc::new(Gazetteer::empty());
|
||||
let events_tx = events::channel();
|
||||
let http_client = reqwest::Client::new();
|
||||
|
||||
let (tx_a, _rx_a) = analyze::channel();
|
||||
let (tx_t, rx_t) = transcribe::channel();
|
||||
let transcribe_busy: WorkerBusy = Arc::new(AtomicBool::new(false));
|
||||
let heal_busy: WorkerBusy = Arc::new(AtomicBool::new(false));
|
||||
|
||||
// Spawn the transcribe worker in the background; it processes jobs
|
||||
// until the channel is closed at test teardown.
|
||||
let worker = tokio::spawn(transcribe::worker::run(
|
||||
rx_t,
|
||||
config.clone(),
|
||||
http_client.clone(),
|
||||
transcribe_busy.clone(),
|
||||
vocab.clone(),
|
||||
events_tx.clone(),
|
||||
));
|
||||
|
||||
let pipeline = PipelineState {
|
||||
analyze_busy: AnalyzeBusy(Arc::new(AtomicBool::new(false))),
|
||||
analyze_tx: tx_a,
|
||||
transcribe_busy: TranscribeBusy(transcribe_busy.clone()),
|
||||
transcribe_tx: tx_t.clone(),
|
||||
oneliner_heal_busy: OnelinerHealBusy(heal_busy.clone()),
|
||||
};
|
||||
|
||||
// Act 1: enqueue the initial job, wait for the 503 failure.
|
||||
tx_t.send(transcribe::TranscribeJob {
|
||||
audio_path: audio.clone(),
|
||||
user_slug: slug.into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Wait until the worker has processed the first job (busy false
|
||||
// AND the 503 mock has recorded a hit). Polling both conditions
|
||||
// avoids a race where the worker hasn't started yet.
|
||||
let start = std::time::Instant::now();
|
||||
loop {
|
||||
let idle = !transcribe_busy.load(Ordering::Acquire);
|
||||
let hits = whisper.received_requests().await.unwrap().len();
|
||||
if idle && hits >= 1 {
|
||||
break;
|
||||
}
|
||||
if start.elapsed() > Duration::from_secs(10) {
|
||||
panic!("worker did not process first job within 10s (idle={idle}, hits={hits})");
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
|
||||
// Invariant after transient failure: .m4a still there, no .failed.
|
||||
assert!(audio.exists(), "transient error must not consume the .m4a");
|
||||
assert!(
|
||||
!case_dir.join("2026-04-15T20-30-00Z.m4a.failed").exists(),
|
||||
".m4a.failed must not exist after transient error"
|
||||
);
|
||||
assert!(
|
||||
!case_dir
|
||||
.join("2026-04-15T20-30-00Z.transcript.txt")
|
||||
.exists(),
|
||||
"no transcript yet — the 503 produced nothing"
|
||||
);
|
||||
|
||||
// Act 2: heal re-enqueues the pending .m4a (second mock → 200).
|
||||
pipeline
|
||||
.heal_orphans_if_idle(&user_root, slug, &http_client, &config, &vocab, &events_tx)
|
||||
.await;
|
||||
|
||||
// Assert: the transcript lands. Poll because the worker runs
|
||||
// asynchronously after the heal hands off the job.
|
||||
let transcript = case_dir.join("2026-04-15T20-30-00Z.transcript.txt");
|
||||
let start = std::time::Instant::now();
|
||||
while !transcript.exists() {
|
||||
if start.elapsed() > Duration::from_secs(10) {
|
||||
panic!(
|
||||
"transcript did not appear within 10s (whisper hits: {})",
|
||||
whisper.received_requests().await.unwrap().len()
|
||||
);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
|
||||
let body = std::fs::read_to_string(&transcript).unwrap();
|
||||
assert_eq!(body, "recovered text");
|
||||
|
||||
// Teardown: close the worker channel so the background task exits.
|
||||
drop(tx_t);
|
||||
drop(pipeline);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), worker).await;
|
||||
}
|
||||
Reference in New Issue
Block a user