Add last_failure to worker state

The `WorkerSnapshot` now includes `last_failure`, which tracks the time
of the last recorded error. This allows the UI to display a more
accurate status, especially when a failure occurs after a recent
success.

The `pick_footer_status` function has been updated to prioritize
`last_failure`. If a failure occurred more recently than the last
success, the footer will show `Offline`, regardless of the success age.
This addresses scenarios where the server might have temporarily failed,
and the UI should reflect that immediately.

The `run_loop` function now updates `last_failure` upon transient or
terminal upload errors and poll failures. This ensures that the new
state information is correctly propagated.
This commit is contained in:
2026-04-18 13:45:21 +02:00
parent 305d739f56
commit 79e91246c6
4 changed files with 144 additions and 47 deletions
+4 -2
View File
@@ -524,7 +524,8 @@ impl DoctateApp {
return; return;
}; };
let snap = worker_rx.borrow().clone(); let snap = worker_rx.borrow().clone();
let last_age = snap.last_success.map(|i| i.elapsed()); let last_success_age = snap.last_success.map(|i| i.elapsed());
let last_failure_age = snap.last_failure.map(|i| i.elapsed());
let threshold = self let threshold = self
.config .config
.as_ref() .as_ref()
@@ -535,7 +536,8 @@ impl DoctateApp {
self.finalizing.is_some(), self.finalizing.is_some(),
snap.state, snap.state,
snap.queue_len, snap.queue_len,
last_age, last_success_age,
last_failure_age,
threshold, threshold,
); );
+113 -38
View File
@@ -61,28 +61,45 @@ pub enum FooterStatus {
/// Decide which footer status to show. Pure function — no side effects, /// Decide which footer status to show. Pure function — no side effects,
/// fully table-testable. Inputs chosen to match what `DoctateApp` can /// fully table-testable. Inputs chosen to match what `DoctateApp` can
/// observe cheaply per egui frame. /// observe cheaply per egui frame.
///
/// Decision rules for the `Idle` branch:
/// - A failure younger than the last success (or a failure with no
/// prior success at all) wins → `Offline`.
/// - Otherwise fall back to success-age vs. threshold (classic staleness).
///
/// `last_*_age` arguments are `Option<Duration>` = "seconds since that
/// event, or `None` if it never happened". Smaller Duration = more recent.
pub fn pick_footer_status( pub fn pick_footer_status(
finalizing: bool, finalizing: bool,
worker_state: WorkerState, worker_state: WorkerState,
queue_len: usize, queue_len: usize,
last_success_age: Option<Duration>, last_success_age: Option<Duration>,
last_failure_age: Option<Duration>,
offline_threshold: Duration, offline_threshold: Duration,
) -> FooterStatus { ) -> FooterStatus {
if finalizing { if finalizing {
return FooterStatus::Finalizing; return FooterStatus::Finalizing;
} }
if queue_len > 0 || worker_state == WorkerState::Uploading { match worker_state {
return FooterStatus::Uploading { WorkerState::Uploading => FooterStatus::Uploading {
count: queue_len.max(1), count: queue_len.max(1),
}; },
} WorkerState::Polling => FooterStatus::Synchronizing,
if worker_state == WorkerState::Polling { WorkerState::Idle => {
return FooterStatus::Synchronizing; let failure_more_recent = match (last_success_age, last_failure_age) {
} (Some(s), Some(f)) => f < s, // smaller age = younger
match last_success_age { (None, Some(_)) => true, // only a failure on record
None => FooterStatus::NeverConnected, _ => false,
Some(age) if age > offline_threshold => FooterStatus::Offline, };
Some(_) => FooterStatus::Online, if failure_more_recent {
return FooterStatus::Offline;
}
match last_success_age {
None => FooterStatus::NeverConnected,
Some(age) if age > offline_threshold => FooterStatus::Offline,
Some(_) => FooterStatus::Online,
}
}
} }
} }
@@ -92,6 +109,16 @@ mod tests {
const THRESHOLD: Duration = Duration::from_secs(30); const THRESHOLD: Duration = Duration::from_secs(30);
/// Convenience: most tests don't care about the failure field.
fn status(
finalizing: bool,
worker_state: WorkerState,
queue_len: usize,
last_success_age: Option<Duration>,
) -> FooterStatus {
pick_footer_status(finalizing, worker_state, queue_len, last_success_age, None, THRESHOLD)
}
#[test] #[test]
fn finalizing_wins_over_everything() { fn finalizing_wins_over_everything() {
let s = pick_footer_status( let s = pick_footer_status(
@@ -99,6 +126,7 @@ mod tests {
WorkerState::Uploading, WorkerState::Uploading,
5, 5,
Some(Duration::from_millis(1)), Some(Duration::from_millis(1)),
Some(Duration::from_millis(1)),
THRESHOLD, THRESHOLD,
); );
assert_eq!(s, FooterStatus::Finalizing); assert_eq!(s, FooterStatus::Finalizing);
@@ -106,84 +134,131 @@ mod tests {
#[test] #[test]
fn uploading_when_queue_non_empty() { fn uploading_when_queue_non_empty() {
let s = pick_footer_status( let s = status(
false, false,
WorkerState::Uploading, WorkerState::Uploading,
3, 3,
Some(Duration::from_millis(1)), Some(Duration::from_millis(1)),
THRESHOLD,
); );
assert_eq!(s, FooterStatus::Uploading { count: 3 }); assert_eq!(s, FooterStatus::Uploading { count: 3 });
} }
#[test] #[test]
fn uploading_reports_at_least_one_even_if_scan_raced() { fn uploading_reports_at_least_one_even_if_scan_raced() {
// Edge case: WorkerState says Uploading, but scan returned 0 let s = status(
// (just finished a file and the next scan hasn't happened yet).
// We still want to show "Uploading" to avoid a flicker back to
// Online right before the next file starts.
let s = pick_footer_status(
false, false,
WorkerState::Uploading, WorkerState::Uploading,
0, 0,
Some(Duration::from_millis(1)), Some(Duration::from_millis(1)),
THRESHOLD,
); );
assert_eq!(s, FooterStatus::Uploading { count: 1 }); assert_eq!(s, FooterStatus::Uploading { count: 1 });
} }
#[test] #[test]
fn synchronizing_when_polling_and_no_queue() { fn synchronizing_when_polling_and_no_queue() {
let s = pick_footer_status( let s = status(false, WorkerState::Polling, 0, Some(Duration::from_millis(1)));
false,
WorkerState::Polling,
0,
Some(Duration::from_millis(1)),
THRESHOLD,
);
assert_eq!(s, FooterStatus::Synchronizing); assert_eq!(s, FooterStatus::Synchronizing);
} }
#[test] #[test]
fn never_connected_without_last_success() { fn never_connected_without_last_success() {
let s = pick_footer_status(false, WorkerState::Idle, 0, None, THRESHOLD); let s = status(false, WorkerState::Idle, 0, None);
assert_eq!(s, FooterStatus::NeverConnected); assert_eq!(s, FooterStatus::NeverConnected);
} }
#[test] #[test]
fn offline_when_last_success_is_stale() { fn offline_when_last_success_is_stale() {
let s = pick_footer_status( let s = status(
false, false,
WorkerState::Idle, WorkerState::Idle,
0, 0,
Some(Duration::from_secs(120)), Some(Duration::from_secs(120)),
THRESHOLD,
); );
assert_eq!(s, FooterStatus::Offline); assert_eq!(s, FooterStatus::Offline);
} }
#[test] #[test]
fn online_when_last_success_is_fresh() { fn online_when_last_success_is_fresh() {
let s = pick_footer_status( let s = status(false, WorkerState::Idle, 0, Some(Duration::from_secs(5)));
false,
WorkerState::Idle,
0,
Some(Duration::from_secs(5)),
THRESHOLD,
);
assert_eq!(s, FooterStatus::Online); assert_eq!(s, FooterStatus::Online);
} }
#[test] #[test]
fn offline_threshold_is_inclusive_of_equal_age_as_online() { fn offline_threshold_is_inclusive_of_equal_age_as_online() {
// Exactly at threshold: still online (strict ">" in logic). let s = status(false, WorkerState::Idle, 0, Some(THRESHOLD));
assert_eq!(s, FooterStatus::Online);
}
#[test]
fn offline_while_queue_has_items_but_worker_backing_off() {
let s = status(
false,
WorkerState::Idle,
3,
Some(Duration::from_secs(120)),
);
assert_eq!(s, FooterStatus::Offline);
}
#[test]
fn online_while_queue_has_items_but_worker_idle_and_recent_success() {
let s = status(false, WorkerState::Idle, 3, Some(Duration::from_secs(2)));
assert_eq!(s, FooterStatus::Online);
}
#[test]
fn never_connected_while_queue_has_items_and_no_success_ever() {
let s = status(false, WorkerState::Idle, 3, None);
assert_eq!(s, FooterStatus::NeverConnected);
}
// --- last_failure-driven scenarios ---
/// Core regression this fix addresses: server was reachable (fresh
/// last_success), then went offline mid-session; a single failed
/// upload must flip the footer to Offline immediately, without
/// waiting for the success-age threshold.
#[test]
fn failure_younger_than_success_forces_offline() {
let s = pick_footer_status(
false,
WorkerState::Idle,
1,
Some(Duration::from_secs(10)), // was online 10s ago
Some(Duration::from_secs(1)), // but failed 1s ago
THRESHOLD,
);
assert_eq!(s, FooterStatus::Offline);
}
/// Complement: success younger than the last failure → the server
/// recovered. Show Online.
#[test]
fn success_younger_than_failure_wins_back_online() {
let s = pick_footer_status( let s = pick_footer_status(
false, false,
WorkerState::Idle, WorkerState::Idle,
0, 0,
Some(THRESHOLD), Some(Duration::from_secs(1)), // succeeded 1s ago
Some(Duration::from_secs(10)), // previous failure 10s ago
THRESHOLD, THRESHOLD,
); );
assert_eq!(s, FooterStatus::Online); assert_eq!(s, FooterStatus::Online);
} }
/// Only a failure ever, no success: treat as Offline, not
/// NeverConnected — we tried and got rejected, which is a stronger
/// signal than "silent cold start".
#[test]
fn only_failure_no_success_is_offline() {
let s = pick_footer_status(
false,
WorkerState::Idle,
0,
None,
Some(Duration::from_secs(5)),
THRESHOLD,
);
assert_eq!(s, FooterStatus::Offline);
}
} }
+26 -6
View File
@@ -65,8 +65,13 @@ pub struct WorkerSnapshot {
/// `Uploading`. /// `Uploading`.
pub queue_len: usize, pub queue_len: usize,
/// Set to `Some(now)` after any successful upload or poll (200/304). /// Set to `Some(now)` after any successful upload or poll (200/304).
/// `None` before the first success. Drives Online/Offline heuristics. /// `None` before the first success. Drives Online/Offline heuristics
/// together with `last_failure`.
pub last_success: Option<Instant>, pub last_success: Option<Instant>,
/// Set to `Some(now)` after any failed upload or poll. Compared to
/// `last_success` in the UI: if `last_failure` is younger, the
/// footer shows Offline regardless of the success's age.
pub last_failure: Option<Instant>,
} }
impl Default for WorkerSnapshot { impl Default for WorkerSnapshot {
@@ -75,6 +80,7 @@ impl Default for WorkerSnapshot {
state: WorkerState::Idle, state: WorkerState::Idle,
queue_len: 0, queue_len: 0,
last_success: None, last_success: None,
last_failure: None,
} }
} }
} }
@@ -197,12 +203,17 @@ async fn run_loop(
let mut backoff = INITIAL_BACKOFF; let mut backoff = INITIAL_BACKOFF;
let mut last_success: Option<Instant> = None; let mut last_success: Option<Instant> = None;
let mut last_failure: Option<Instant> = None;
let publish = |state: WorkerState, queue_len: usize, last: Option<Instant>| { let publish = |state: WorkerState,
queue_len: usize,
last_ok: Option<Instant>,
last_fail: Option<Instant>| {
let snap = WorkerSnapshot { let snap = WorkerSnapshot {
state, state,
queue_len, queue_len,
last_success: last, last_success: last_ok,
last_failure: last_fail,
}; };
let _ = snapshot_tx.send(snap); let _ = snapshot_tx.send(snap);
}; };
@@ -216,7 +227,7 @@ async fn run_loop(
let next = files.into_iter().next(); let next = files.into_iter().next();
if let Some(upload) = next { if let Some(upload) = next {
publish(WorkerState::Uploading, queue_len, last_success); publish(WorkerState::Uploading, queue_len, last_success, last_failure);
let _ = events_tx.send(UploadEvent::Started(upload.case_id)); let _ = events_tx.send(UploadEvent::Started(upload.case_id));
match post_upload(&http, &config, &upload).await { match post_upload(&http, &config, &upload).await {
UploadOutcome::Succeeded(status) => { UploadOutcome::Succeeded(status) => {
@@ -235,17 +246,24 @@ async fn run_loop(
} }
UploadOutcome::Transient(reason) => { UploadOutcome::Transient(reason) => {
warn!(case_id = %upload.case_id, reason = %reason, "upload failed transiently"); warn!(case_id = %upload.case_id, reason = %reason, "upload failed transiently");
last_failure = Some(Instant::now());
let _ = events_tx.send(UploadEvent::Failed { let _ = events_tx.send(UploadEvent::Failed {
case_id: upload.case_id, case_id: upload.case_id,
reason, reason,
will_retry: true, will_retry: true,
}); });
// Flip back to Idle before the backoff sleep so the UI
// doesn't misreport "Uploading" during a passive wait.
// The footer derives "Offline" from `last_failure` being
// younger than `last_success`.
publish(WorkerState::Idle, queue_len, last_success, last_failure);
tokio::time::sleep(backoff).await; tokio::time::sleep(backoff).await;
backoff = (backoff * 2).min(MAX_BACKOFF); backoff = (backoff * 2).min(MAX_BACKOFF);
} }
UploadOutcome::Terminal(reason) => { UploadOutcome::Terminal(reason) => {
warn!(case_id = %upload.case_id, reason = %reason, "upload failed terminally — deleting files"); warn!(case_id = %upload.case_id, reason = %reason, "upload failed terminally — deleting files");
delete_pair(&upload.file).await; delete_pair(&upload.file).await;
last_failure = Some(Instant::now());
backoff = INITIAL_BACKOFF; backoff = INITIAL_BACKOFF;
let _ = events_tx.send(UploadEvent::Failed { let _ = events_tx.send(UploadEvent::Failed {
case_id: upload.case_id, case_id: upload.case_id,
@@ -255,7 +273,7 @@ async fn run_loop(
} }
} }
} else { } else {
publish(WorkerState::Polling, 0, last_success); publish(WorkerState::Polling, 0, last_success, last_failure);
match poll_once(&http, &config, &store, &cache, &mut etag).await { match poll_once(&http, &config, &store, &cache, &mut etag).await {
Ok(PollOutcome::Success) | Ok(PollOutcome::NotModified) => { Ok(PollOutcome::Success) | Ok(PollOutcome::NotModified) => {
let now = Instant::now(); let now = Instant::now();
@@ -264,13 +282,15 @@ async fn run_loop(
} }
Ok(PollOutcome::TransientHttp(code)) => { Ok(PollOutcome::TransientHttp(code)) => {
warn!(code, "poll returned transient HTTP error"); warn!(code, "poll returned transient HTTP error");
last_failure = Some(Instant::now());
} }
Err(e) => { Err(e) => {
warn!(error = %e, "poll error"); warn!(error = %e, "poll error");
last_failure = Some(Instant::now());
} }
} }
publish(WorkerState::Idle, 0, last_success); publish(WorkerState::Idle, 0, last_success, last_failure);
// Wait for next tick OR kick. Cancel-safe on both arms. // Wait for next tick OR kick. Cancel-safe on both arms.
tokio::select! { tokio::select! {
+1 -1
View File
@@ -7,7 +7,7 @@ use tracing::debug;
/// the transcript itself (no hallucination) so the doctor can recognize the /// the transcript itself (no hallucination) so the doctor can recognize the
/// case from the watch face. English instructions for small-model /// case from the watch face. English instructions for small-model
/// robustness, German output. /// robustness, German output.
const SYSTEM_PROMPT: &str = "You are a keyword-extraction assistant for a German medical dictation app. Given a transcript, output a short German one-line label for the case.\n\nCRITICAL — empty-string rule:\nIf the transcript is empty, silent, or contains no identifiable medical keyword, respond with an EMPTY STRING. Do not guess, do not produce filler, do not write \"Unbekannt\" or \"Kein Befund\". An empty answer is correct.\n\nRules:\n- 2-4 words, at most 50 characters, single line.\n- Start with the most important noun from the transcript: body region, organ, imaging modality, or key finding.\n- Use ONLY terms that appear in the transcript. Do not invent, translate, or paraphrase. Do not add diagnoses the doctor did not mention.\n- No filler prefixes: \"Patient\", \"Fall\", \"Verdacht auf\", \"Untersuchung\", \"Diktat\", articles.\n- No intensifiers (\"massiv\", \"ausgeprägt\", \"deutlich\", \"hochgradig\").\n- No quotes, no trailing period, no introductory phrase.\n- If the doctor explicitly names the case (\"Bezeichnung: X\", \"Fall-ID: X\"), start with that exact string.\n- Respond in German. One line only.\n\nExamples:\n\nTranscript: \"Bezeichnung: Schulter links. Patient klagt über...\"\nOutput: Schulter links\n\nTranscript: \"...V.a. Bandscheibenvorfall in der LWS, Schmerzen strahlen...\"\nOutput: LWS, V.a. Bandscheibenvorfall\n\nREMINDER: For silent, empty, or non-medical transcripts, respond with nothing at all — an empty string. Never invent content."; const SYSTEM_PROMPT: &str = "You are a keyword-extraction assistant for a German medical dictation app. Given a transcript, output a short German one-line label for the case.\n\nCRITICAL — silence rule:\nIf the transcript is empty, silent, or contains no identifiable medical keyword, return no text at all. Leave your answer completely blank. Do NOT echo the words 'empty', 'nothing', 'none', 'N/A', 'null', '-', a placeholder, or any description of emptiness. Emit zero characters.\n\nRules:\n- 2-4 words, at most 50 characters, single line.\n- Start with the most important noun from the transcript: body region, organ, imaging modality, or key finding.\n- Use ONLY terms that appear in the transcript. Do not invent, translate, or paraphrase. Do not add diagnoses the doctor did not mention.\n- No filler prefixes: \"Patient\", \"Fall\", \"Verdacht auf\", \"Untersuchung\", \"Diktat\", articles.\n- No intensifiers (\"massiv\", \"ausgeprägt\", \"deutlich\", \"hochgradig\").\n- No quotes, no trailing period, no introductory phrase.\n- If the doctor explicitly names the case (\"Bezeichnung: X\", \"Fall-ID: X\"), start with that exact string.\n- Respond in German. One line only.\n\nExamples:\n\nTranscript: \"Bezeichnung: Schulter links. Patient klagt über...\"\nOutput: Schulter links\n\nTranscript: \"...V.a. Bandscheibenvorfall in der LWS, Schmerzen strahlen...\"\nOutput: LWS, V.a. Bandscheibenvorfall\n\nREMINDER: For silent, empty, or non-medical transcripts, emit zero characters. Do not write the words 'empty', 'nothing', 'none', or any placeholder — just produce no output.";
/// Deterministic leading-filler prefixes stripped after the LLM response. /// Deterministic leading-filler prefixes stripped after the LLM response.
/// The trailing space is part of the match — we only strip whole tokens. /// The trailing space is part of the match — we only strip whole tokens.