//! Unified client-side worker that owns the only outbound HTTP channel: //! it uploads pending recordings and polls `/api/oneliners` from a single //! `reqwest::Client`, in a single `tokio::select!` loop. //! //! Design constraints (see plan `berpr-fe-den-plan-lively-journal.md`): //! //! - **Uploads have priority.** If anything sits in `pending/`, the worker //! uploads it; polling happens only when the directory is empty. //! - **Disk is the queue.** No in-memory `VecDeque` — the worker scans //! `pending/` at the top of every iteration. FIFO via `recorded_at`. //! - **Kick channel, no payload.** `notify()` on the handle just wakes //! the `select!` sleep; the next iteration finds the fresh file on //! disk. //! - **Backoff is a local `Duration`.** Exponential, resets on success, //! capped at `MAX_BACKOFF`. Not persisted across restarts. //! - **Terminal upload failures delete the file.** 401/413/etc. will //! never succeed; keeping the audio around would violate the //! data-minimization rule. use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex as StdMutex}; use std::time::{Duration, Instant}; use doctate_common::ack::{AckResponse, AckStatus}; use doctate_common::constants::{ API_KEY_HEADER, CONTENT_TYPE_AUDIO_MP4, FIELD_AUDIO, FIELD_CASE_ID, FIELD_RECORDED_AT, UPLOAD_PATH, }; use doctate_common::oneliners::{ONELINERS_PATH, OnelinersResponse}; use reqwest::StatusCode; use tokio::sync::{mpsc, watch}; use tokio::task::AbortHandle; use tracing::{debug, info, warn}; use uuid::Uuid; use crate::case_store::CaseStore; use crate::snapshot_cache::{CachedSnapshot, SnapshotCache}; use crate::upload::{PendingUpload, UploadEvent, scan_pending_dir, sidecar_path_for}; const INITIAL_BACKOFF: Duration = Duration::from_secs(2); const MAX_BACKOFF: Duration = Duration::from_secs(60); #[derive(Debug, Clone)] pub struct SyncConfig { pub server_url: String, pub api_key: String, pub poll_interval: Duration, pub window_hours: u32, } /// Three-state observable: what the worker is currently doing. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum WorkerState { Idle, Uploading, Polling, } /// Single observable the UI reads each frame. `watch::channel` semantics: /// only the last value matters, readers see either the newest or nothing. #[derive(Debug, Clone)] pub struct WorkerSnapshot { pub state: WorkerState, /// Result of the last `scan_pending_dir` at the top of a loop /// iteration. `0` during `Polling` / `Idle`, positive during /// `Uploading`. pub queue_len: usize, /// Set of `case_id`s that currently have at least one file sitting /// in the pending directory. Shared as an `Arc` so the UI can clone /// the borrow cheaply and look up per-case without scanning disk. pub pending_case_ids: Arc>, /// Set to `Some(now)` after any successful upload or poll (200/304). /// `None` before the first success. Drives Online/Offline heuristics /// together with `last_failure`. pub last_success: Option, /// Set to `Some(now)` after any failed upload or poll. Compared to /// `last_success` in the UI: if `last_failure` is younger, the /// footer shows Offline regardless of the success's age. pub last_failure: Option, } impl Default for WorkerSnapshot { fn default() -> Self { Self { state: WorkerState::Idle, queue_len: 0, pending_case_ids: Arc::new(HashSet::new()), last_success: None, last_failure: None, } } } /// Handle to the worker task. Drop aborts the task; outstanding disk /// artefacts survive and are picked up by the next worker instance. pub struct ServerSync { abort: AbortHandle, kick_tx: mpsc::UnboundedSender<()>, snapshot_rx: watch::Receiver, /// Kept in an `Arc>` so tests can peek at `last_success` /// independently of the watch channel. Not required for production. #[allow(dead_code)] last_success_mirror: Arc>>, } impl ServerSync { /// Spawn the worker task. Returns the handle plus the event receiver /// for upload lifecycle events (started / succeeded / failed). /// /// Must run inside a Tokio runtime context. pub fn spawn( http: Arc, config: SyncConfig, store: Arc, cache: Arc, pending_dir: PathBuf, ) -> (Self, mpsc::UnboundedReceiver) { let (kick_tx, kick_rx) = mpsc::unbounded_channel::<()>(); let (events_tx, events_rx) = mpsc::unbounded_channel::(); let (snapshot_tx, snapshot_rx) = watch::channel(WorkerSnapshot::default()); let last_success_mirror = Arc::new(StdMutex::new(None)); let task = tokio::spawn(run_loop( http, config, store, cache, pending_dir, kick_rx, events_tx, snapshot_tx, last_success_mirror.clone(), )); let handle = Self { abort: task.abort_handle(), kick_tx, snapshot_rx, last_success_mirror, }; (handle, events_rx) } /// Wake the worker if it's sleeping in the poll-idle `select!`. Used /// by the app thread after writing a fresh sidecar to `pending/`. /// Failure to send means the worker is already dropped — fine, /// ignore. pub fn notify(&self) { let _ = self.kick_tx.send(()); } /// Cheap clone of the observable for the UI. Each call yields an /// independent receiver — all receivers see the same `borrow()`. pub fn snapshot(&self) -> watch::Receiver { self.snapshot_rx.clone() } } impl Drop for ServerSync { fn drop(&mut self) { self.abort.abort(); } } /// Outcome classification for a single upload attempt. pub(crate) enum UploadOutcome { Succeeded(AckStatus), Transient(String), Terminal(String), } /// Outcome classification for a single poll cycle. Symmetric to the old /// `PollOutcome` but folded into the unified module. #[derive(Debug, PartialEq, Eq)] pub(crate) enum PollOutcome { Success, NotModified, /// Non-2xx non-304 HTTP; logged as transient, retried next tick. TransientHttp(u16), } #[allow(clippy::too_many_arguments)] async fn run_loop( http: Arc, config: SyncConfig, store: Arc, cache: Arc, pending_dir: PathBuf, mut kick_rx: mpsc::UnboundedReceiver<()>, events_tx: mpsc::UnboundedSender, snapshot_tx: watch::Sender, last_success_mirror: Arc>>, ) { info!( interval_s = config.poll_interval.as_secs(), window_h = config.window_hours, "server sync worker started" ); // Prime the store + ETag from disk cache before the first request. let mut etag: Option = None; if let Some(cached) = cache.load().await { if let Err(e) = store.merge_server_snapshot(&cached.response).await { warn!(error = %e, "priming store from cache failed"); } etag = Some(cached.etag); debug!("store primed from cache"); } let mut backoff = INITIAL_BACKOFF; let mut last_success: Option = None; let mut last_failure: Option = None; let publish = |state: WorkerState, queue_len: usize, pending_ids: Arc>, last_ok: Option, last_fail: Option| { let snap = WorkerSnapshot { state, queue_len, pending_case_ids: pending_ids, last_success: last_ok, last_failure: last_fail, }; let _ = snapshot_tx.send(snap); }; loop { // Top of iteration: re-scan disk. Cheap when the directory is // empty; one stat per file when it isn't. let mut files = scan_pending_dir(&pending_dir); files.sort_by(|a, b| a.recorded_at.cmp(&b.recorded_at)); // FIFO let queue_len = files.len(); let pending_ids: Arc> = Arc::new(files.iter().map(|u| u.case_id).collect()); let next = files.into_iter().next(); if let Some(upload) = next { publish( WorkerState::Uploading, queue_len, pending_ids.clone(), last_success, last_failure, ); let _ = events_tx.send(UploadEvent::Started(upload.case_id)); match post_upload(&http, &config, &upload).await { UploadOutcome::Succeeded(status) => { delete_pair(&upload.file).await; if let Err(e) = store.mark_synced(upload.case_id).await { warn!(error = %e, case_id = %upload.case_id, "mark_synced failed"); } let now = Instant::now(); last_success = Some(now); *last_success_mirror.lock().expect("poisoned") = Some(now); backoff = INITIAL_BACKOFF; let _ = events_tx.send(UploadEvent::Succeeded { case_id: upload.case_id, status, }); } UploadOutcome::Transient(reason) => { warn!(case_id = %upload.case_id, reason = %reason, "upload failed transiently"); last_failure = Some(Instant::now()); let _ = events_tx.send(UploadEvent::Failed { case_id: upload.case_id, reason, will_retry: true, }); // Flip back to Idle before the backoff sleep so the UI // doesn't misreport "Uploading" during a passive wait. // The footer derives "Offline" from `last_failure` being // younger than `last_success`. publish( WorkerState::Idle, queue_len, pending_ids.clone(), last_success, last_failure, ); tokio::time::sleep(backoff).await; backoff = (backoff * 2).min(MAX_BACKOFF); } UploadOutcome::Terminal(reason) => { warn!(case_id = %upload.case_id, reason = %reason, "upload failed terminally — deleting files"); delete_pair(&upload.file).await; last_failure = Some(Instant::now()); backoff = INITIAL_BACKOFF; let _ = events_tx.send(UploadEvent::Failed { case_id: upload.case_id, reason, will_retry: false, }); } } } else { publish( WorkerState::Polling, 0, pending_ids.clone(), last_success, last_failure, ); match poll_once(&http, &config, &store, &cache, &mut etag).await { Ok(PollOutcome::Success) | Ok(PollOutcome::NotModified) => { let now = Instant::now(); last_success = Some(now); *last_success_mirror.lock().expect("poisoned") = Some(now); } Ok(PollOutcome::TransientHttp(code)) => { warn!(code, "poll returned transient HTTP error"); last_failure = Some(Instant::now()); } Err(e) => { warn!(error = %e, "poll error"); last_failure = Some(Instant::now()); } } publish( WorkerState::Idle, 0, pending_ids.clone(), last_success, last_failure, ); // Wait for next tick OR kick. Cancel-safe on both arms. tokio::select! { _ = tokio::time::sleep(config.poll_interval) => {} _ = kick_rx.recv() => {} } } } } /// Perform one upload attempt. Error classification follows the rules: /// - I/O errors reading the audio file → Terminal (file is broken, retry /// won't help). /// - `reqwest::send` errors → Transient (network). /// - 2xx with parseable ACK → Succeeded. /// - 408/429/5xx → Transient (server will likely recover). /// - Other 4xx → Terminal (401, 413, 415, …; operator must fix). pub(crate) async fn post_upload( http: &reqwest::Client, config: &SyncConfig, upload: &PendingUpload, ) -> UploadOutcome { let audio = match tokio::fs::read(&upload.file).await { Ok(bytes) => bytes, Err(e) => return UploadOutcome::Terminal(format!("read audio file: {e}")), }; let file_name = upload .file .file_name() .map(|s| s.to_string_lossy().into_owned()) .unwrap_or_else(|| "recording.m4a".to_string()); let audio_part = match reqwest::multipart::Part::bytes(audio) .file_name(file_name) .mime_str(CONTENT_TYPE_AUDIO_MP4) { Ok(p) => p, Err(e) => return UploadOutcome::Terminal(format!("build multipart: {e}")), }; let form = reqwest::multipart::Form::new() .text(FIELD_CASE_ID, upload.case_id.to_string()) .text(FIELD_RECORDED_AT, upload.recorded_at.clone()) .part(FIELD_AUDIO, audio_part); let url = format!("{}{}", config.server_url.trim_end_matches('/'), UPLOAD_PATH); let resp = match http .post(&url) .header(API_KEY_HEADER, &config.api_key) .multipart(form) .send() .await { Ok(r) => r, Err(e) => return UploadOutcome::Transient(format!("network: {e}")), }; let status = resp.status(); if status.is_success() { match resp.json::().await { Ok(ack) => UploadOutcome::Succeeded(ack.status), Err(e) => UploadOutcome::Terminal(format!("parse ack: {e}")), } } else if status.as_u16() == 408 || status.as_u16() == 429 || status.is_server_error() { let body = resp.text().await.unwrap_or_default(); UploadOutcome::Transient(format!("HTTP {status}: {body}")) } else { let body = resp.text().await.unwrap_or_default(); UploadOutcome::Terminal(format!("HTTP {status}: {body}")) } } /// Perform one poll of `/api/oneliners`. Side effects: on 200 merge into /// the store and persist the response to the cache; update `etag` on 200. pub(crate) async fn poll_once( http: &reqwest::Client, config: &SyncConfig, store: &CaseStore, cache: &SnapshotCache, etag: &mut Option, ) -> Result { let url = format!( "{}{}?hours={}", config.server_url.trim_end_matches('/'), ONELINERS_PATH, config.window_hours ); let mut req = http.get(&url).header(API_KEY_HEADER, &config.api_key); if let Some(tag) = etag.as_deref() { req = req.header("If-None-Match", tag); } let resp = req.send().await?; let status = resp.status(); match status { StatusCode::OK => { let new_etag = extract_etag(&resp); let body: OnelinersResponse = resp.json().await?; // Merge adds/updates markers the server still knows about; // reconcile removes the ones it no longer does (within the // window reported by `as_of` / `window_hours`). Running both // after every successful poll keeps the client UI in step // with server-side deletions without any extra round-trips. store.merge_server_snapshot(&body).await?; store.reconcile_with_server_snapshot(&body).await?; if let Some(tag) = new_etag.clone() { let cached = CachedSnapshot { etag: tag.clone(), fetched_at: doctate_common::now_rfc3339(), response: body, }; if let Err(e) = cache.store(&cached).await { warn!(error = %e, "snapshot cache write failed"); } *etag = Some(tag); } else { warn!("server returned 200 without ETag — conditional requests disabled"); *etag = None; } Ok(PollOutcome::Success) } StatusCode::NOT_MODIFIED => Ok(PollOutcome::NotModified), other => Ok(PollOutcome::TransientHttp(other.as_u16())), } } #[derive(Debug, thiserror::Error)] pub enum PollError { #[error("http: {0}")] Http(#[from] reqwest::Error), #[error("deserialize response: {0}")] Parse(#[from] serde_json::Error), #[error("case store: {0}")] Store(#[from] crate::case_store::CaseStoreError), } fn extract_etag(resp: &reqwest::Response) -> Option { resp.headers() .get("etag") .and_then(|v| v.to_str().ok()) .map(str::to_owned) } async fn delete_pair(m4a: &Path) { if let Err(e) = tokio::fs::remove_file(m4a).await { warn!(path = %m4a.display(), error = %e, "delete m4a failed"); } let sidecar = sidecar_path_for(m4a); if let Err(e) = tokio::fs::remove_file(&sidecar).await { warn!(path = %sidecar.display(), error = %e, "delete sidecar failed"); } } #[cfg(test)] mod tests { use super::*; use crate::upload::write_sidecar; use doctate_common::oneliners::OnelinerEntry; use tempfile::TempDir; use time::format_description::well_known::Rfc3339; use uuid::Uuid; use wiremock::matchers::{header, method, path as wpath}; use wiremock::{Mock, MockServer, ResponseTemplate}; const TEST_KEY: &str = "test-key-sync"; const TEST_ETAG: &str = "W/\"42-72\""; fn cfg(server: &MockServer) -> SyncConfig { SyncConfig { server_url: server.uri(), api_key: TEST_KEY.into(), poll_interval: Duration::from_millis(50), window_hours: 72, } } fn ack_body(status: AckStatus) -> AckResponse { AckResponse { case_id: "550e8400-e29b-41d4-a716-446655440000".into(), recorded_at: "2026-04-18T10:32:00Z".into(), status, } } fn oneliner_body(case_id: &str, oneliner: Option<&str>) -> OnelinersResponse { OnelinersResponse { as_of: "2026-04-18T10:00:00Z".into(), window_hours: 72, oneliners: vec![OnelinerEntry { case_id: case_id.into(), oneliner: oneliner.map(str::to_owned), created_at: "2026-04-18T09:30:00Z".into(), last_recording_at: Some("2026-04-18T09:45:00Z".into()), updated_at: Some("2026-04-18T09:45:00Z".into()), }], } } fn sample_pending(dir: &Path, case_id: Uuid, recorded_at: &str) -> PendingUpload { let stem = format!("{case_id}_{}", recorded_at.replace([':', '-'], "-")); let file = dir.join(format!("{stem}.m4a")); std::fs::write(&file, b"dummy audio").unwrap(); let upload = PendingUpload { case_id, recorded_at: recorded_at.into(), file, }; write_sidecar(&upload).unwrap(); upload } async fn make_store_and_cache(tmp: &TempDir) -> (Arc, Arc) { let (store, _rx) = CaseStore::open(tmp.path().join("cases")).await.unwrap(); let cache = SnapshotCache::new(tmp.path().join("cache.json")); (Arc::new(store), Arc::new(cache)) } // --- unit-level tests on post_upload / poll_once --- #[tokio::test] async fn post_upload_succeeds_on_200_with_ack() { let server = MockServer::start().await; Mock::given(method("POST")) .and(wpath(UPLOAD_PATH)) .and(header(API_KEY_HEADER, TEST_KEY)) .respond_with(ResponseTemplate::new(200).set_body_json(ack_body(AckStatus::Received))) .mount(&server) .await; let tmp = TempDir::new().unwrap(); let upload = sample_pending(tmp.path(), Uuid::new_v4(), "2026-04-18T10-32-00Z"); let http = reqwest::Client::new(); let outcome = post_upload(&http, &cfg(&server), &upload).await; assert!(matches!( outcome, UploadOutcome::Succeeded(AckStatus::Received) )); } #[tokio::test] async fn post_upload_transient_on_500() { let server = MockServer::start().await; Mock::given(method("POST")) .respond_with(ResponseTemplate::new(500).set_body_string("boom")) .mount(&server) .await; let tmp = TempDir::new().unwrap(); let upload = sample_pending(tmp.path(), Uuid::new_v4(), "2026-04-18T10-32-00Z"); let http = reqwest::Client::new(); assert!(matches!( post_upload(&http, &cfg(&server), &upload).await, UploadOutcome::Transient(_) )); } #[tokio::test] async fn post_upload_terminal_on_401() { let server = MockServer::start().await; Mock::given(method("POST")) .respond_with(ResponseTemplate::new(401).set_body_string("nope")) .mount(&server) .await; let tmp = TempDir::new().unwrap(); let upload = sample_pending(tmp.path(), Uuid::new_v4(), "2026-04-18T10-32-00Z"); let http = reqwest::Client::new(); assert!(matches!( post_upload(&http, &cfg(&server), &upload).await, UploadOutcome::Terminal(_) )); } /// End-to-end reconcile: a synced marker that the server omits from /// its next snapshot must vanish locally after a single poll. This /// is the scenario the user reported ("deleted cases still show up /// on the client") — the wiring is: `poll_once` → `merge` → `reconcile`. #[tokio::test] async fn poll_once_reconciles_missing_synced_marker() { let server = MockServer::start().await; Mock::given(method("GET")) .and(wpath(ONELINERS_PATH)) .respond_with( ResponseTemplate::new(200) .insert_header("etag", TEST_ETAG) // Authoritative snapshot with zero entries — any // synced local marker inside the window is deleted // server-side. .set_body_json(OnelinersResponse { as_of: "2026-04-18T10:00:00Z".into(), window_hours: 72, oneliners: vec![], }), ) .mount(&server) .await; let tmp = TempDir::new().unwrap(); let (store, cache) = make_store_and_cache(&tmp).await; // Seed a synced marker inside the window. let stale_id = Uuid::new_v4(); store .create_local( stale_id, time::OffsetDateTime::parse("2026-04-18T09:00:00Z", &Rfc3339).unwrap(), ) .await .unwrap(); store.mark_synced(stale_id).await.unwrap(); assert_eq!(store.list().await.len(), 1); // Poll once — the merge produces no add/update, reconcile drops // the stale marker. let http = reqwest::Client::new(); let mut etag: Option = None; let outcome = poll_once(&http, &cfg(&server), &store, &cache, &mut etag) .await .unwrap(); assert_eq!(outcome, PollOutcome::Success); assert!( store.list().await.is_empty(), "reconcile must remove the synced marker the server no longer lists" ); } #[tokio::test] async fn poll_once_200_merges_and_caches() { let server = MockServer::start().await; let case_id = "550e8400-e29b-41d4-a716-000000000001"; Mock::given(method("GET")) .and(wpath(ONELINERS_PATH)) .respond_with( ResponseTemplate::new(200) .insert_header("etag", TEST_ETAG) .set_body_json(oneliner_body(case_id, Some("K"))), ) .mount(&server) .await; let tmp = TempDir::new().unwrap(); let (store, cache) = make_store_and_cache(&tmp).await; let http = reqwest::Client::new(); let mut etag: Option = None; let outcome = poll_once(&http, &cfg(&server), &store, &cache, &mut etag) .await .unwrap(); assert_eq!(outcome, PollOutcome::Success); assert_eq!(etag.as_deref(), Some(TEST_ETAG)); assert_eq!(store.list().await.len(), 1); assert!(cache.load().await.is_some()); } // --- high-level spawn/loop tests --- /// Startup scan: files left in pending_dir before spawn must be /// uploaded without any explicit submit call. #[tokio::test] async fn startup_scan_picks_up_leftover_files() { let server = MockServer::start().await; Mock::given(method("POST")) .and(wpath(UPLOAD_PATH)) .respond_with(ResponseTemplate::new(200).set_body_json(ack_body(AckStatus::Received))) .mount(&server) .await; // A catch-all for the idle poll so wiremock doesn't 404 after upload. Mock::given(method("GET")) .and(wpath(ONELINERS_PATH)) .respond_with(ResponseTemplate::new(304).insert_header("etag", TEST_ETAG)) .mount(&server) .await; let tmp = TempDir::new().unwrap(); let pending_dir = tmp.path().join("pending"); std::fs::create_dir_all(&pending_dir).unwrap(); let case_id = Uuid::new_v4(); sample_pending(&pending_dir, case_id, "2026-04-18T10-00-00Z"); let (store, cache) = make_store_and_cache(&tmp).await; store .create_local(case_id, time::OffsetDateTime::now_utc()) .await .unwrap(); let (sync, mut events) = ServerSync::spawn( Arc::new(reqwest::Client::new()), cfg(&server), store.clone(), cache, pending_dir.clone(), ); // Expect Started + Succeeded within a short window. let mut saw_started = false; let mut saw_success = false; for _ in 0..10 { let Ok(Some(ev)) = tokio::time::timeout(Duration::from_millis(500), events.recv()).await else { continue; }; match ev { UploadEvent::Started(_) => saw_started = true, UploadEvent::Succeeded { .. } => { saw_success = true; break; } UploadEvent::Failed { reason, .. } => panic!("unexpected failure: {reason}"), } } assert!(saw_started); assert!(saw_success); // Disk-side: pair must be gone, marker must be synced. let leftovers = scan_pending_dir(&pending_dir); assert!(leftovers.is_empty(), "files must be deleted after ACK"); assert!( store.list().await.iter().any(|m| m.synced_to_server), "mark_synced must have flipped the flag" ); drop(sync); } /// Polling only happens when the pending dir is empty. With an item /// present, `GET /api/oneliners` must not fire before the upload /// completes (enforced by a mock with `.expect(0)`). #[tokio::test] async fn poll_deferred_while_pending_upload() { let server = MockServer::start().await; // Upload always succeeds. Mock::given(method("POST")) .and(wpath(UPLOAD_PATH)) .respond_with(ResponseTemplate::new(200).set_body_json(ack_body(AckStatus::Received))) .expect(1) .mount(&server) .await; // No GET on /api/oneliners is allowed until after the upload — // wiremock's `.expect(0)` verifies this when `server` drops. Mock::given(method("GET")) .and(wpath(ONELINERS_PATH)) .respond_with(ResponseTemplate::new(304).insert_header("etag", TEST_ETAG)) .expect(0..) .mount(&server) .await; let tmp = TempDir::new().unwrap(); let pending_dir = tmp.path().join("pending"); std::fs::create_dir_all(&pending_dir).unwrap(); sample_pending(&pending_dir, Uuid::new_v4(), "2026-04-18T10-00-00Z"); let (store, cache) = make_store_and_cache(&tmp).await; let (sync, mut events) = ServerSync::spawn( Arc::new(reqwest::Client::new()), cfg(&server), store, cache, pending_dir, ); // Wait for the single Succeeded event. loop { match tokio::time::timeout(Duration::from_millis(500), events.recv()).await { Ok(Some(UploadEvent::Succeeded { .. })) => break, Ok(Some(_)) => continue, Ok(None) => panic!("event channel closed"), Err(_) => panic!("timed out waiting for Succeeded"), } } drop(sync); // Mock expectations are checked on `server` drop at end of test. } /// Terminal 401: file + sidecar must be deleted, no retry loop. #[tokio::test] async fn terminal_401_deletes_files_no_retry() { let server = MockServer::start().await; Mock::given(method("POST")) .and(wpath(UPLOAD_PATH)) .respond_with(ResponseTemplate::new(401).set_body_string("nope")) .expect(1) .mount(&server) .await; Mock::given(method("GET")) .and(wpath(ONELINERS_PATH)) .respond_with(ResponseTemplate::new(304).insert_header("etag", TEST_ETAG)) .mount(&server) .await; let tmp = TempDir::new().unwrap(); let pending_dir = tmp.path().join("pending"); std::fs::create_dir_all(&pending_dir).unwrap(); let upload = sample_pending(&pending_dir, Uuid::new_v4(), "2026-04-18T10-00-00Z"); let (store, cache) = make_store_and_cache(&tmp).await; let (sync, mut events) = ServerSync::spawn( Arc::new(reqwest::Client::new()), cfg(&server), store, cache, pending_dir.clone(), ); // Expect a Failed{will_retry:false} event. let mut saw_terminal = false; for _ in 0..10 { if let Ok(Some(UploadEvent::Failed { will_retry: false, .. })) = tokio::time::timeout(Duration::from_millis(500), events.recv()).await { saw_terminal = true; break; } } assert!(saw_terminal, "expected terminal failure event"); // Files gone. assert!(!upload.file.exists()); assert!(!sidecar_path_for(&upload.file).exists()); drop(sync); } /// Kick wakes the worker out of the idle sleep. With a long /// poll_interval, the only way an upload happens quickly after a /// submit is via the kick channel. #[tokio::test] async fn notify_kick_wakes_idle_worker() { let server = MockServer::start().await; Mock::given(method("POST")) .and(wpath(UPLOAD_PATH)) .respond_with(ResponseTemplate::new(200).set_body_json(ack_body(AckStatus::Received))) .mount(&server) .await; Mock::given(method("GET")) .and(wpath(ONELINERS_PATH)) .respond_with(ResponseTemplate::new(304).insert_header("etag", TEST_ETAG)) .mount(&server) .await; let tmp = TempDir::new().unwrap(); let pending_dir = tmp.path().join("pending"); std::fs::create_dir_all(&pending_dir).unwrap(); let (store, cache) = make_store_and_cache(&tmp).await; let mut slow_cfg = cfg(&server); slow_cfg.poll_interval = Duration::from_secs(60); // effectively never ticks let (sync, mut events) = ServerSync::spawn( Arc::new(reqwest::Client::new()), slow_cfg, store, cache, pending_dir.clone(), ); // Give the worker a moment to settle into idle-sleep. tokio::time::sleep(Duration::from_millis(100)).await; // Now drop a file in and kick the worker. sample_pending(&pending_dir, Uuid::new_v4(), "2026-04-18T10-00-00Z"); sync.notify(); // Without the kick, we'd wait 60s. With it, we see an event fast. let success = async { loop { if let Some(UploadEvent::Succeeded { .. }) = events.recv().await { break; } } }; tokio::time::timeout(Duration::from_secs(3), success) .await .expect("kick should have woken the worker well within 3s"); drop(sync); } /// FIFO order by `recorded_at`: oldest file uploads first. #[tokio::test] async fn fifo_order_by_recorded_at() { let server = MockServer::start().await; // Record which case_ids we saw, in order. let seen: Arc>> = Arc::new(tokio::sync::Mutex::new(Vec::new())); let seen_for_mock = seen.clone(); let responder = wiremock::ResponseTemplate::new(200).set_body_json(ack_body(AckStatus::Received)); // Simpler: just let both POSTs succeed; infer order from event // stream on our side. let _ = seen_for_mock; let _ = responder; Mock::given(method("POST")) .and(wpath(UPLOAD_PATH)) .respond_with(ResponseTemplate::new(200).set_body_json(ack_body(AckStatus::Received))) .mount(&server) .await; Mock::given(method("GET")) .and(wpath(ONELINERS_PATH)) .respond_with(ResponseTemplate::new(304).insert_header("etag", TEST_ETAG)) .mount(&server) .await; let tmp = TempDir::new().unwrap(); let pending_dir = tmp.path().join("pending"); std::fs::create_dir_all(&pending_dir).unwrap(); let older = Uuid::new_v4(); let newer = Uuid::new_v4(); // Create newer first, older second — the disk order should NOT // determine upload order; `recorded_at` should. sample_pending(&pending_dir, newer, "2026-04-18T11-00-00Z"); sample_pending(&pending_dir, older, "2026-04-18T10-00-00Z"); let (store, cache) = make_store_and_cache(&tmp).await; let (sync, mut events) = ServerSync::spawn( Arc::new(reqwest::Client::new()), cfg(&server), store, cache, pending_dir, ); let mut ok_order: Vec = Vec::new(); while ok_order.len() < 2 { match tokio::time::timeout(Duration::from_millis(500), events.recv()).await { Ok(Some(UploadEvent::Succeeded { case_id, .. })) => ok_order.push(case_id), Ok(Some(_)) => continue, _ => panic!("timed out waiting for two Succeeded events"), } } assert_eq!(ok_order, vec![older, newer], "FIFO by recorded_at"); drop(sync); } #[tokio::test] async fn abort_on_drop_leaves_files_on_disk() { let server = MockServer::start().await; // Slow POST so we can abort mid-flight. Mock::given(method("POST")) .and(wpath(UPLOAD_PATH)) .respond_with( ResponseTemplate::new(200) .set_delay(Duration::from_secs(5)) .set_body_json(ack_body(AckStatus::Received)), ) .mount(&server) .await; Mock::given(method("GET")) .and(wpath(ONELINERS_PATH)) .respond_with(ResponseTemplate::new(304).insert_header("etag", TEST_ETAG)) .mount(&server) .await; let tmp = TempDir::new().unwrap(); let pending_dir = tmp.path().join("pending"); std::fs::create_dir_all(&pending_dir).unwrap(); let upload = sample_pending(&pending_dir, Uuid::new_v4(), "2026-04-18T10-00-00Z"); let (store, cache) = make_store_and_cache(&tmp).await; let (sync, _events) = ServerSync::spawn( Arc::new(reqwest::Client::new()), cfg(&server), store, cache, pending_dir.clone(), ); // Let the worker enter the slow POST. tokio::time::sleep(Duration::from_millis(200)).await; drop(sync); tokio::time::sleep(Duration::from_millis(100)).await; // Both files must still be on disk. assert!(upload.file.exists(), "m4a must remain after abort"); assert!( sidecar_path_for(&upload.file).exists(), "sidecar must remain after abort" ); } }