fix: separate transient from permanent transcribe failures
Before, every Whisper error (5xx, timeout, Minerva down, corrupt audio,
4xx) renamed `<ts>.m4a` to `<ts>.m4a.failed` uniformly, turning transient
outages into permanent sackgassen that only a manual admin reset could
undo. Cases whose every recording was `.m4a.failed` stuck without a
persisted oneliner state; the UI masked them via a fallback in
`compute_oneliner_display`.
The core insight: a recoverable error is not an error. Transient Whisper
failures now leave the audio as plain `.m4a` so the existing page-load
heal (`enqueue_pending_for_user`) re-enqueues it on the next refresh —
which is exactly what happens when Minerva comes back. No new sidecar,
no retry count, no scheduler.
Changes:
- `WhisperError::is_transient` classifies `Http`, 5xx, 408, 429 as
transient; `Io` and other 4xx as permanent.
- Transcribe worker: transient → info log + continue (audio stays .m4a);
permanent → `mark_failed` as before.
- `has_any_transcript` counts `.m4a.failed` as terminal, so
`update_oneliner` runs for failed-only cases and settles
`OnelinerState::Empty` (analogous to the silent-only fix in 4531f85).
- New verdrängender `fehler`-Badge (#c00) in case list and detail when
at least one `.m4a.failed` exists; recording-level message shortened
to plain "Transkription fehlgeschlagen".
Tests:
- New integration: transient 503 leaves `.m4a` intact, heal recovers.
- New integration: `.m4a.failed`-only case settles to
`OnelinerState::Empty` without calling Ollama.
- New unit: `is_transient` table test across relevant status codes.
- New unit: `has_any_transcript` returns true for `.m4a.failed`-only
case and false for pending-only case.
- Existing worker test retargeted from 500 to 400 and renamed; added
companion `worker_leaves_m4a_intact_on_transient_whisper_error`.
This commit is contained in:
@@ -71,6 +71,12 @@ struct UserCaseView {
|
||||
recordings_count: usize,
|
||||
analyzing: bool,
|
||||
has_document: bool,
|
||||
/// True iff at least one recording is `.m4a.failed` — a *permanent*
|
||||
/// transcribe failure (ffmpeg remux or Whisper 4xx). Transient Whisper
|
||||
/// errors never set this flag because they leave the file as plain
|
||||
/// `.m4a` for the page-load heal to retry. Drives the verdrängende
|
||||
/// `fehler`-Badge over `offen`/`ausgewertet`.
|
||||
has_failed_recording: bool,
|
||||
/// True iff the inline "Analysieren"-button should be enabled.
|
||||
/// Requires: not currently analyzing, all non-failed recordings
|
||||
/// transcribed, and an LLM is configured.
|
||||
@@ -213,6 +219,10 @@ struct CasePageTemplate {
|
||||
llm_missing: bool,
|
||||
analyzing: bool,
|
||||
has_document: bool,
|
||||
/// True iff at least one `.m4a.failed` recording exists. Drives the
|
||||
/// verdrängende `fehler`-Badge in the case header; see the same
|
||||
/// field on `UserCaseView` for semantics.
|
||||
has_failed_recording: bool,
|
||||
is_admin: bool,
|
||||
/// True when the case carries a `.closed` marker. Toggles the
|
||||
/// header action button between close and reopen, and appends
|
||||
@@ -491,6 +501,7 @@ pub async fn handle_case_page(
|
||||
.iter()
|
||||
.filter(|r| r.transcript.has_file())
|
||||
.count();
|
||||
let has_failed_recording = recordings.iter().any(|r| r.failed);
|
||||
let case_id_short = case_id_str.chars().take(8).collect();
|
||||
let is_admin = user.is_admin();
|
||||
|
||||
@@ -515,6 +526,7 @@ pub async fn handle_case_page(
|
||||
llm_missing: flags.llm_missing,
|
||||
analyzing: flags.analyzing,
|
||||
has_document: flags.has_document,
|
||||
has_failed_recording,
|
||||
is_admin,
|
||||
is_closed,
|
||||
}
|
||||
@@ -732,6 +744,7 @@ async fn compute_case_view(
|
||||
.unwrap_or_default();
|
||||
let recordings_count = recordings.len();
|
||||
let non_failed_count = recordings.iter().filter(|r| !r.failed).count();
|
||||
let has_failed_recording = recordings.iter().any(|r| r.failed);
|
||||
let all_transcribed = non_failed_count > 0
|
||||
&& recordings
|
||||
.iter()
|
||||
@@ -757,6 +770,7 @@ async fn compute_case_view(
|
||||
recordings_count,
|
||||
analyzing,
|
||||
has_document,
|
||||
has_failed_recording,
|
||||
can_analyze,
|
||||
is_closed,
|
||||
days_until_purge,
|
||||
|
||||
@@ -156,16 +156,26 @@ pub(crate) async fn cases_needing_oneliner_in(user_root: &Path) -> Vec<PathBuf>
|
||||
out
|
||||
}
|
||||
|
||||
/// True iff at least one `.transcript.txt` sidecar exists in `case_dir`
|
||||
/// — i.e., at least one recording is either [`TranscriptState::Silent`]
|
||||
/// or [`TranscriptState::Content`]. Cases where every recording is
|
||||
/// still `Pending` return false: there's nothing for the oneliner
|
||||
/// worker to act on yet.
|
||||
/// True iff the case has at least one recording in a *terminal* state —
|
||||
/// meaning the transcribe pipeline will produce no further artefacts
|
||||
/// and `update_oneliner` has enough information to settle the case.
|
||||
///
|
||||
/// Silent transcripts count here because `update_oneliner` needs to
|
||||
/// settle them to `OnelinerState::Empty`. The previous implementation
|
||||
/// required *content* specifically, which left silent-only cases stuck
|
||||
/// with no oneliner state — and the UI stuck on "Generating".
|
||||
/// Three terminal shapes count:
|
||||
/// - `.transcript.txt` with content → [`TranscriptState::Content`]
|
||||
/// - `.transcript.txt` empty → [`TranscriptState::Silent`]
|
||||
/// - `.m4a.failed` → permanent transcription failure (ffmpeg remux or
|
||||
/// Whisper 4xx). Transient Whisper errors leave the file as plain
|
||||
/// `.m4a` without sidecar (Pending), so they do *not* match here —
|
||||
/// the page-load heal re-enqueues them on the next refresh.
|
||||
///
|
||||
/// Cases where every recording is still `Pending` (plain `.m4a`, no
|
||||
/// sidecar, not marked failed) return false: the transcriber hasn't
|
||||
/// finished yet and there is nothing for the oneliner worker to act
|
||||
/// on.
|
||||
///
|
||||
/// Both Silent and permanent-failed count because `update_oneliner`
|
||||
/// settles each to [`OnelinerState::Empty`] via the same no-content
|
||||
/// branch in `worker::update_oneliner`.
|
||||
async fn has_any_transcript(case_dir: &Path) -> bool {
|
||||
let Ok(mut files) = tokio::fs::read_dir(case_dir).await else {
|
||||
return false;
|
||||
@@ -173,7 +183,7 @@ async fn has_any_transcript(case_dir: &Path) -> bool {
|
||||
while let Ok(Some(f)) = files.next_entry().await {
|
||||
if f.file_name()
|
||||
.to_str()
|
||||
.is_some_and(|s| s.ends_with(".transcript.txt"))
|
||||
.is_some_and(|s| s.ends_with(".transcript.txt") || s.ends_with(".m4a.failed"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
@@ -356,4 +366,42 @@ mod tests {
|
||||
|
||||
assert!(cases_needing_oneliner(data.path()).await.is_empty());
|
||||
}
|
||||
|
||||
/// `.m4a.failed`-only cases are terminal — the pipeline will produce
|
||||
/// no further artefacts. Recovery must pick them up so
|
||||
/// `update_oneliner` can persist `OnelinerState::Empty`; otherwise
|
||||
/// the case would rely on the UI fallback forever and never have a
|
||||
/// stable on-disk state.
|
||||
#[tokio::test]
|
||||
async fn cases_needing_oneliner_picks_up_failed_only_cases() {
|
||||
let data = tempdir().unwrap();
|
||||
let case = data.path().join("user").join("case1");
|
||||
tokio::fs::create_dir_all(&case).await.unwrap();
|
||||
tokio::fs::write(case.join("2026-04-16T10-00-00Z.m4a.failed"), b"x")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let got = cases_needing_oneliner(data.path()).await;
|
||||
assert_eq!(got, vec![(case, "user".to_owned())]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn has_any_transcript_true_for_failed_only_case() {
|
||||
let case = tempdir().unwrap();
|
||||
tokio::fs::write(case.path().join("2026-04-16T10-00-00Z.m4a.failed"), b"x")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(has_any_transcript(case.path()).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn has_any_transcript_false_for_pending_only_case() {
|
||||
let case = tempdir().unwrap();
|
||||
tokio::fs::write(case.path().join("2026-04-16T10-00-00Z.m4a"), b"audio")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!has_any_transcript(case.path()).await);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,35 @@ impl std::fmt::Display for WhisperError {
|
||||
|
||||
impl std::error::Error for WhisperError {}
|
||||
|
||||
impl WhisperError {
|
||||
/// Classify this error as transient (retry will likely succeed later) or
|
||||
/// permanent (retrying the same input will hit the same wall).
|
||||
///
|
||||
/// Rationale: the transcribe worker uses this to decide whether to leave
|
||||
/// the `.m4a` in place (page-load heal re-enqueues it) or rename it to
|
||||
/// `.m4a.failed` (terminal; only a manual reset un-fails it).
|
||||
///
|
||||
/// - `Http`: reqwest wraps all transport issues (DNS, connect, TLS, read,
|
||||
/// timeout, mid-flight reset) — every one of these is a candidate for
|
||||
/// „Minerva is back in a minute", so treat as transient.
|
||||
/// - `Status`: 5xx is server-side trouble; 408 (Request Timeout) and 429
|
||||
/// (Too Many Requests) are explicit retry signals. Everything else in
|
||||
/// the 4xx range (400 Bad Request, 415 Unsupported Media, ...) means
|
||||
/// the payload itself is the problem and replays will fail identically.
|
||||
/// - `Io`: local filesystem error while reading the audio file. Either
|
||||
/// the file is gone or the disk is misbehaving; a retry of the same
|
||||
/// path is not the right response.
|
||||
pub fn is_transient(&self) -> bool {
|
||||
match self {
|
||||
Self::Http(_) => true,
|
||||
Self::Status { status, .. } => {
|
||||
*status == 408 || *status == 429 || (500..=599).contains(status)
|
||||
}
|
||||
Self::Io(_) => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// POST the audio file to whisper-asr-webservice and return the transcript text.
|
||||
///
|
||||
/// `output=txt` → plain-text response. `language` pins the language so the model
|
||||
@@ -97,3 +126,48 @@ pub async fn transcribe(
|
||||
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Table-driven classification test. `Http` variant is not publicly
|
||||
/// constructible (reqwest owns its error ctor), so the transient-Http
|
||||
/// path is covered by the integration test in
|
||||
/// `tests/transient_failure_retries_test.rs`. Same code path is exercised
|
||||
/// here via a 503 Status, which also routes to `is_transient == true`.
|
||||
#[test]
|
||||
fn is_transient_classifies_status_codes() {
|
||||
let cases: &[(u16, bool)] = &[
|
||||
(400, false),
|
||||
(401, false),
|
||||
(403, false),
|
||||
(404, false),
|
||||
(408, true),
|
||||
(415, false),
|
||||
(422, false),
|
||||
(429, true),
|
||||
(500, true),
|
||||
(502, true),
|
||||
(503, true),
|
||||
(504, true),
|
||||
];
|
||||
for &(status, expected) in cases {
|
||||
let err = WhisperError::Status {
|
||||
status,
|
||||
body: String::new(),
|
||||
};
|
||||
assert_eq!(
|
||||
err.is_transient(),
|
||||
expected,
|
||||
"status {status} expected is_transient={expected}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_transient_io_is_permanent() {
|
||||
let err = WhisperError::Io(std::io::Error::other("disk gone"));
|
||||
assert!(!err.is_transient());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,7 +78,20 @@ pub async fn run(
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
error!(audio = %audio_path.display(), error = %e, "whisper call failed");
|
||||
mark_failed(&audio_path, &events_tx, &job.user_slug).await;
|
||||
if e.is_transient() {
|
||||
// Leave the recording as plain `.m4a` (no `.failed` suffix)
|
||||
// so the page-load heal (`enqueue_pending_for_user`) picks
|
||||
// it up on the next refresh. The natural retry trigger is
|
||||
// a human navigating the UI — no scheduler, no sidecar.
|
||||
info!(
|
||||
audio = %audio_path.display(),
|
||||
error = %e,
|
||||
reason = "transient",
|
||||
"recording left as .m4a, page-load heal will re-enqueue"
|
||||
);
|
||||
} else {
|
||||
mark_failed(&audio_path, &events_tx, &job.user_slug).await;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user