From 66b3b7e4c8eeed13c1fddaa8bfbb665bf1f25040 Mon Sep 17 00:00:00 2001 From: Brummel Date: Mon, 27 Apr 2026 12:48:25 +0200 Subject: [PATCH] Refactor recording metadata to JSON sidecar Replaces the `.transcript.txt` sidecar with a structured `.json` file for recording metadata. This change consolidates transcript text, duration, and other potential metadata into a single, extensible JSON object. This also refactors the `TranscriptState` enum to better represent the on-disk state (absence of file means pending) and the in-memory representation. The `Transcript` enum now specifically models the terminal outcomes of the transcriber (`Silent` or `Content`). The commit includes updates to documentation, data structures, path handling, and various tests to align with the new metadata format. --- docs/projektplan.md | 6 +- doctate-common/src/lib.rs | 2 +- doctate-common/src/recordings.rs | 214 +++++++++++++----- server/src/analyze/auto_trigger.rs | 60 +++-- server/src/paths.rs | 129 +++++++++-- server/src/routes/case_actions.rs | 39 +++- server/src/routes/upload.rs | 2 +- server/src/routes/web.rs | 85 ++++--- server/src/transcribe/recovery.rs | 99 ++++---- server/src/transcribe/worker.rs | 125 +++++----- server/templates/case_recordings.html | 4 +- server/tests/analyze_test.rs | 10 +- server/tests/common/mod.rs | 4 +- server/tests/common/seed.rs | 65 +++++- server/tests/delete_recording_test.rs | 20 +- server/tests/oneliner_heal_decoupled_test.rs | 21 +- server/tests/oneliner_override_test.rs | 26 ++- server/tests/silent_case_empty_test.rs | 6 +- server/tests/transcribe_test.rs | 31 ++- .../tests/transient_failure_retries_test.rs | 24 +- 20 files changed, 637 insertions(+), 335 deletions(-) diff --git a/docs/projektplan.md b/docs/projektplan.md index ca7eb73..bcce5a5 100644 --- a/docs/projektplan.md +++ b/docs/projektplan.md @@ -48,12 +48,12 @@ Ubuntu Server (RTX 3060, 12 GB VRAM) ### Filesystem ist Source of Truth (SoT) -Der Zustand eines Falls wird **ausschließlich aus dem Dateisystem** abgeleitet, nicht aus einer parallelen Sidecar- oder Metadaten-Datei. Das Vorhandensein bestimmter Dateimarker (`.m4a`, `.m4a.failed`, `.transcript.txt`, `oneliner.json`, `analysis_input.json`, `document.md`, `.deleted`) plus die `WorkerBusy`-Flags bestimmen jederzeit eindeutig, was als Nächstes zu tun ist. +Der Zustand eines Falls wird **ausschließlich aus dem Dateisystem** abgeleitet, nicht aus einer parallelen Sidecar- oder Metadaten-Datei. Das Vorhandensein bestimmter Dateimarker (`.m4a`, `.m4a.failed`, `.json` (Per-Recording-Metadaten — Transcript + Duration), `oneliner.json`, `analysis_input.json`, `document.md`, `.closed`) plus die `WorkerBusy`-Flags bestimmen jederzeit eindeutig, was als Nächstes zu tun ist. **Konsequenzen**: - Keine State-Duplizierung (kein `state.json`, keine DB). Der zu synchronisierende Zweitstand fehlt ersatzlos — also kann er auch nicht drift. -- Recovery-Scan nach Restart ist trivial: finde `.m4a` ohne `.transcript.txt` → transkribieren. Natürliche Idempotenz, kein Crash-Recovery-Protokoll. -- Crash mitten in Whisper = `.transcript.txt` fehlt = nächster Lauf macht's nochmal. Keine Intermediate-States, die rückwärts abgewickelt werden müssten. +- Recovery-Scan nach Restart ist trivial: finde `.m4a` ohne `.json` → transkribieren. Natürliche Idempotenz, kein Crash-Recovery-Protokoll. +- Crash mitten in Whisper = `.json` fehlt = nächster Lauf macht's nochmal. Keine Intermediate-States, die rückwärts abgewickelt werden müssten. Single atomic write (`tmp` + `rename`) am Ende der Pipeline garantiert: entweder gibt es ein vollständiges JSON oder keines — nie ein halbes. - Manuelle Reparatur möglich: Datei löschen, neu triggern. Kein „state irgendwie auf Received setzen". **Bewusst verzichtet** auf: retry-Budget, Failed-Kategorie mit Error-Details. Wenn ein Audio dauerhaft an Whisper scheitert, loggt der Server im Crash-Loop — das fällt sofort auf und wird manuell entfernt. Diese Kategorie wird erst eingeführt, wenn ein realer Bedarf entsteht (z.B. hochvolumiger Betrieb). diff --git a/doctate-common/src/lib.rs b/doctate-common/src/lib.rs index 6afb778..0d30587 100644 --- a/doctate-common/src/lib.rs +++ b/doctate-common/src/lib.rs @@ -13,6 +13,6 @@ pub use constants::{ UPLOAD_PATH, }; pub use oneliners::{ONELINER_FILENAME, ONELINERS_PATH, OnelinerEntry, OnelinersResponse}; -pub use recordings::TranscriptState; +pub use recordings::{RECORDING_META_SUFFIX, RecordingMeta, Transcript, TranscriptState}; pub use timestamp::{filename_stem_to_recorded_at, now_rfc3339, recorded_at_to_filename_stem}; pub use url::join_url; diff --git a/doctate-common/src/recordings.rs b/doctate-common/src/recordings.rs index 5522f65..1add151 100644 --- a/doctate-common/src/recordings.rs +++ b/doctate-common/src/recordings.rs @@ -1,56 +1,108 @@ //! Recording-side domain types shared between server and clients. //! -//! The recording lifecycle on disk has three disjoint states for the -//! `.transcript.txt` sidecar, which is easy to conflate because the -//! filesystem collapses two of them onto "file exists": +//! Each recording has a single sidecar JSON next to its audio file: +//! `.json` (suffix [`RECORDING_META_SUFFIX`]). It carries the +//! container duration and the transcript outcome in one structured +//! document. //! -//! 1. `Pending` — no file yet, the transcriber hasn't run (or hasn't -//! finished) for this recording. -//! 2. `Silent` — file exists but is empty / whitespace-only. The -//! transcriber classified the audio as silence. Terminal: the next -//! transcribe pass will not overwrite it unless the user re-uploads. -//! 3. `Content` — file exists with real text. +//! On-disk vs. in-memory representation: +//! +//! - [`Transcript`] is the **on-disk** shape. It only models the two +//! *terminal* outcomes the transcriber can persist (`Silent` or +//! `Content`). A recording that has not been transcribed yet has no +//! sidecar at all — there is no "pending" variant on disk. +//! - [`TranscriptState`] is the **in-memory** shape returned by the +//! server's reader. It adds a [`TranscriptState::Pending`] variant +//! for "no sidecar exists", so callers can `match` on the three +//! logically distinct states without having to inspect the +//! filesystem themselves. //! //! Call-sites that collapse this onto a 2-state view (`Option`, //! `.exists()`, `is_empty()`) drift apart and have caused real bugs: //! e.g. the UI treating `Silent` like `Content` while the oneliner //! worker treated it like `Pending`, leaving cases stuck in //! "Generating" forever. -//! -//! `TranscriptState` is the canonical representation. It is parse-only -//! (no I/O) so this crate stays dependency-free; the server does the -//! actual file read via `server::paths::read_transcript_state`. -/// Three-way state of a recording's `.transcript.txt` sidecar. +use serde::{Deserialize, Serialize}; + +/// Filename suffix of the per-recording metadata JSON sidecar. +/// +/// Used as `.` next to the `.m4a`, +/// e.g. `2026-04-19T10-00-00Z.json`. Centralised here so the server +/// reader, worker writer, and recovery scan agree on one location. +pub const RECORDING_META_SUFFIX: &str = "json"; + +/// On-disk transcript outcome. +/// +/// Internally tagged: serializes as `{"state": "silent"}` or +/// `{"state": "content", "text": "..."}` — the discriminator lives +/// inside the JSON object next to the data, so the wire format stays +/// flat. Only the two terminal variants the transcriber can produce +/// appear here; "not transcribed yet" is represented by the absence +/// of the sidecar file (see [`TranscriptState::Pending`]). +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "state", rename_all = "lowercase")] +pub enum Transcript { + /// Transcriber ran and classified the audio as silence — terminal. + /// The next transcribe pass will not re-run unless the user + /// re-uploads. + Silent, + /// Transcriber produced non-empty text. + Content { text: String }, +} + +/// Per-recording metadata persisted as `.json`. +/// +/// Single atomic write at the end of the transcribe pipeline. The +/// presence of this file is the canonical signal "the transcriber +/// finished for this recording". Absent file = still pending. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct RecordingMeta { + /// Transcriber outcome. See [`Transcript`]. + pub transcript: Transcript, + /// Container duration in whole seconds, from `ffprobe` on the + /// remuxed audio. `None` is allowed so test fixtures and recovery + /// flows that mint a metadata file without invoking ffprobe stay + /// valid; production worker writes always populate this. + pub duration_seconds: Option, +} + +/// Three-way state of a recording's transcript, as seen by the server +/// after consulting the on-disk sidecar. /// /// Introduced to force every call-site to decide explicitly how it -/// treats `Silent` recordings via `match` exhaustiveness. +/// treats `Silent` recordings via `match` exhaustiveness. Unlike +/// [`Transcript`], this carries a [`Self::Pending`] variant — the +/// reader synthesises that when the sidecar JSON does not exist. #[derive(Debug, Clone, PartialEq, Eq)] pub enum TranscriptState { - /// No `.transcript.txt` file yet — transcriber hasn't produced - /// a result. Callers should typically wait or enqueue. + /// No sidecar JSON yet — transcriber hasn't produced a result. + /// Callers should typically wait or enqueue. Pending, - /// File exists but is empty or whitespace-only. Terminal state: - /// the transcriber already ran and classified this recording as - /// silence. Treat as "transcribed, no medical content". + /// JSON exists with `state: "silent"`. Terminal: the transcriber + /// already ran and classified this recording as silence. Silent, - /// File exists with non-empty content. - Content(String), + /// JSON exists with `state: "content"`. + Content { text: String }, +} + +impl From for TranscriptState { + fn from(t: Transcript) -> Self { + match t { + Transcript::Silent => Self::Silent, + Transcript::Content { text } => Self::Content { text }, + } + } } impl TranscriptState { - /// Build a `TranscriptState` from raw read-result bytes. - /// - /// - `None` (file missing / read error) → [`Self::Pending`]. - /// - `Some(s)` with `s.trim().is_empty()` → [`Self::Silent`]. - /// - `Some(s)` otherwise → [`Self::Content`] (stores the full, - /// untrimmed content — downstream joiners may want the original - /// whitespace around the text). - pub fn from_raw(raw: Option<&str>) -> Self { - match raw { + /// Build a `TranscriptState` from a parsed [`RecordingMeta`] (or its + /// absence). `None` (file missing or malformed) becomes + /// [`Self::Pending`]. + pub fn from_meta(meta: Option<&RecordingMeta>) -> Self { + match meta { None => Self::Pending, - Some(s) if s.trim().is_empty() => Self::Silent, - Some(s) => Self::Content(s.to_owned()), + Some(m) => m.transcript.clone().into(), } } @@ -61,9 +113,9 @@ impl TranscriptState { !matches!(self, Self::Pending) } - /// True when there is no file yet and the transcriber still owes - /// us a result. Use this for "should I enqueue transcription?" or - /// "is a oneliner worth deferring?" gates. + /// True when there is no sidecar yet and the transcriber still + /// owes us a result. Use this for "should I enqueue transcription?" + /// or "is a oneliner worth deferring?" gates. pub fn is_pending(&self) -> bool { matches!(self, Self::Pending) } @@ -73,7 +125,7 @@ impl TranscriptState { /// `match` directly. pub fn as_content(&self) -> Option<&str> { match self { - Self::Content(s) => Some(s.as_str()), + Self::Content { text } => Some(text.as_str()), _ => None, } } @@ -84,38 +136,35 @@ mod tests { use super::*; #[test] - fn missing_file_is_pending() { - assert_eq!(TranscriptState::from_raw(None), TranscriptState::Pending); + fn missing_meta_is_pending() { + assert_eq!(TranscriptState::from_meta(None), TranscriptState::Pending); } #[test] - fn empty_string_is_silent() { - assert_eq!(TranscriptState::from_raw(Some("")), TranscriptState::Silent); - } - - #[test] - fn whitespace_only_is_silent() { + fn silent_meta_maps_to_silent_state() { + let meta = RecordingMeta { + transcript: Transcript::Silent, + duration_seconds: Some(7), + }; assert_eq!( - TranscriptState::from_raw(Some(" \n\t \n")), + TranscriptState::from_meta(Some(&meta)), TranscriptState::Silent ); } #[test] - fn text_is_content() { + fn content_meta_maps_to_content_state() { + let meta = RecordingMeta { + transcript: Transcript::Content { + text: "Hallo".into(), + }, + duration_seconds: None, + }; assert_eq!( - TranscriptState::from_raw(Some("Hallo")), - TranscriptState::Content("Hallo".to_owned()) - ); - } - - #[test] - fn text_with_trailing_whitespace_preserved() { - // Downstream join logic may rely on the original bytes; - // from_raw keeps them verbatim. - assert_eq!( - TranscriptState::from_raw(Some("Hallo Welt.\n")), - TranscriptState::Content("Hallo Welt.\n".to_owned()) + TranscriptState::from_meta(Some(&meta)), + TranscriptState::Content { + text: "Hallo".into() + } ); } @@ -123,13 +172,56 @@ mod tests { fn has_file_distinguishes_pending() { assert!(!TranscriptState::Pending.has_file()); assert!(TranscriptState::Silent.has_file()); - assert!(TranscriptState::Content("x".into()).has_file()); + assert!(TranscriptState::Content { text: "x".into() }.has_file()); } #[test] fn as_content_only_for_content() { assert_eq!(TranscriptState::Pending.as_content(), None); assert_eq!(TranscriptState::Silent.as_content(), None); - assert_eq!(TranscriptState::Content("x".into()).as_content(), Some("x")); + assert_eq!( + TranscriptState::Content { text: "x".into() }.as_content(), + Some("x") + ); + } + + #[test] + fn transcript_silent_serializes_with_tag() { + let json = serde_json::to_string(&Transcript::Silent).unwrap(); + assert_eq!(json, r#"{"state":"silent"}"#); + } + + #[test] + fn transcript_content_serializes_with_tag_and_text() { + let json = serde_json::to_string(&Transcript::Content { + text: "Hallo".into(), + }) + .unwrap(); + assert_eq!(json, r#"{"state":"content","text":"Hallo"}"#); + } + + #[test] + fn recording_meta_roundtrip_with_duration() { + let meta = RecordingMeta { + transcript: Transcript::Content { + text: "Befund.".into(), + }, + duration_seconds: Some(42), + }; + let json = serde_json::to_string(&meta).unwrap(); + let back: RecordingMeta = serde_json::from_str(&json).unwrap(); + assert_eq!(back, meta); + } + + #[test] + fn recording_meta_roundtrip_without_duration() { + let meta = RecordingMeta { + transcript: Transcript::Silent, + duration_seconds: None, + }; + let json = serde_json::to_string(&meta).unwrap(); + assert!(json.contains(r#""duration_seconds":null"#)); + let back: RecordingMeta = serde_json::from_str(&json).unwrap(); + assert_eq!(back, meta); } } diff --git a/server/src/analyze/auto_trigger.rs b/server/src/analyze/auto_trigger.rs index cfe3cfc..29cb720 100644 --- a/server/src/analyze/auto_trigger.rs +++ b/server/src/analyze/auto_trigger.rs @@ -318,6 +318,31 @@ mod tests { fs::write(path, b"").await.unwrap(); } + /// Seed a `.json` next to a (presumed already-touched) + /// `.m4a`. Variant is `Transcript::Silent` because these + /// tests historically used an empty `.transcript.txt`, which + /// mapped to `TranscriptState::Silent`. Tests that need real + /// content call `seed_meta_with` with the desired variant. + async fn seed_meta(case_dir: &std::path::Path, stem: &str) { + crate::paths::write_recording_meta_sync( + case_dir, + stem, + doctate_common::Transcript::Silent, + None, + ); + } + + async fn seed_meta_with(case_dir: &std::path::Path, stem: &str, text: &str) { + crate::paths::write_recording_meta_sync( + case_dir, + stem, + doctate_common::Transcript::Content { + text: text.to_owned(), + }, + None, + ); + } + fn set_mtime(path: &std::path::Path, t: SystemTime) { set_file_mtime(path, FileTime::from_system_time(t)).unwrap(); } @@ -335,7 +360,7 @@ mod tests { async fn missing_transcript_skips() { let dir = TempDir::new().unwrap(); touch(&dir.path().join("2026-01-01T10-00-00Z.m4a")).await; - touch(&dir.path().join("2026-01-01T10-00-00Z.transcript.txt")).await; + seed_meta(dir.path(), "2026-01-01T10-00-00Z").await; touch(&dir.path().join("2026-01-01T11-00-00Z.m4a")).await; assert_eq!( evaluate_case(dir.path()).await, @@ -347,7 +372,7 @@ mod tests { async fn all_transcribed_no_document_enqueues() { let dir = TempDir::new().unwrap(); touch(&dir.path().join("2026-01-01T10-00-00Z.m4a")).await; - touch(&dir.path().join("2026-01-01T10-00-00Z.transcript.txt")).await; + seed_meta(dir.path(), "2026-01-01T10-00-00Z").await; assert_eq!(evaluate_case(dir.path()).await, AutoDecision::Enqueue); } @@ -356,7 +381,7 @@ mod tests { let dir = TempDir::new().unwrap(); let m4a = dir.path().join("2026-01-01T10-00-00Z.m4a"); touch(&m4a).await; - touch(&dir.path().join("2026-01-01T10-00-00Z.transcript.txt")).await; + seed_meta(dir.path(), "2026-01-01T10-00-00Z").await; let doc = dir.path().join(DOCUMENT_FILE); touch(&doc).await; @@ -375,7 +400,7 @@ mod tests { let dir = TempDir::new().unwrap(); let m4a = dir.path().join("2026-01-01T10-00-00Z.m4a"); touch(&m4a).await; - touch(&dir.path().join("2026-01-01T10-00-00Z.transcript.txt")).await; + seed_meta(dir.path(), "2026-01-01T10-00-00Z").await; let doc = dir.path().join(DOCUMENT_FILE); touch(&doc).await; @@ -390,7 +415,7 @@ mod tests { async fn analysis_input_present_skips() { let dir = TempDir::new().unwrap(); touch(&dir.path().join("2026-01-01T10-00-00Z.m4a")).await; - touch(&dir.path().join("2026-01-01T10-00-00Z.transcript.txt")).await; + seed_meta(dir.path(), "2026-01-01T10-00-00Z").await; touch(&dir.path().join(ANALYSIS_INPUT_FILE)).await; assert_eq!( evaluate_case(dir.path()).await, @@ -403,7 +428,7 @@ mod tests { let dir = TempDir::new().unwrap(); let m4a = dir.path().join("2026-01-01T10-00-00Z.m4a"); touch(&m4a).await; - touch(&dir.path().join("2026-01-01T10-00-00Z.transcript.txt")).await; + seed_meta(dir.path(), "2026-01-01T10-00-00Z").await; let base = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000); set_mtime(&m4a, base); @@ -431,7 +456,7 @@ mod tests { let dir = TempDir::new().unwrap(); let m4a = dir.path().join("2026-01-01T10-00-00Z.m4a"); touch(&m4a).await; - touch(&dir.path().join("2026-01-01T10-00-00Z.transcript.txt")).await; + seed_meta(dir.path(), "2026-01-01T10-00-00Z").await; let base = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000); set_mtime(&m4a, base); @@ -478,12 +503,12 @@ mod tests { fs::write(eligible.join("2026-01-01T10-00-00Z.m4a"), b"fake") .await .unwrap(); - fs::write( - eligible.join("2026-01-01T10-00-00Z.transcript.txt"), - b"patient mit brustschmerz", + seed_meta_with( + &eligible, + "2026-01-01T10-00-00Z", + "patient mit brustschmerz", ) - .await - .unwrap(); + .await; // Not eligible: has a fresh document. let current_id = "22222222-2222-2222-2222-222222222222"; @@ -491,12 +516,7 @@ mod tests { fs::create_dir_all(¤t).await.unwrap(); let m4a = current.join("2026-01-01T10-00-00Z.m4a"); fs::write(&m4a, b"fake").await.unwrap(); - fs::write( - current.join("2026-01-01T10-00-00Z.transcript.txt"), - b"stable", - ) - .await - .unwrap(); + seed_meta_with(¤t, "2026-01-01T10-00-00Z", "stable").await; let doc = current.join(DOCUMENT_FILE); fs::write(&doc, b"doc").await.unwrap(); let base = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000); @@ -509,9 +529,7 @@ mod tests { fs::write(junk.join("2026-01-01T10-00-00Z.m4a"), b"fake") .await .unwrap(); - fs::write(junk.join("2026-01-01T10-00-00Z.transcript.txt"), b"x") - .await - .unwrap(); + seed_meta_with(&junk, "2026-01-01T10-00-00Z", "x").await; let (tx, mut rx) = analyze_channel(); let events_tx = events_channel(); diff --git a/server/src/paths.rs b/server/src/paths.rs index 656dca1..55b93f4 100644 --- a/server/src/paths.rs +++ b/server/src/paths.rs @@ -1,13 +1,47 @@ use std::path::{Path, PathBuf}; -use doctate_common::TranscriptState; use doctate_common::oneliners::{ONELINER_FILENAME, OnelinerState}; +use doctate_common::{RECORDING_META_SUFFIX, RecordingMeta, TranscriptState}; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use tracing::info; use crate::oneliner_locks::OnelinerLocks; +/// Atomically write `value` as JSON to `final_path`. +/// +/// Writes to `.tmp` first, then `rename`s into place. +/// `rename` within the same filesystem is atomic on every POSIX target +/// we run on, so concurrent readers either see the previous full file +/// or the new full file — never a half-written one. Caller is +/// responsible for the destination's parent directory existing. +/// +/// Used by every metadata sidecar (oneliner state, per-recording meta, +/// …) so the on-disk format never has partial documents the parser +/// would choke on. +pub async fn atomic_write_json(final_path: &Path, value: &T) -> std::io::Result<()> { + let tmp_path = match final_path.file_name() { + Some(name) => { + let mut tmp_name = name.to_owned(); + tmp_name.push(".tmp"); + final_path.with_file_name(tmp_name) + } + // No file_name on a path means we cannot derive a sibling tmp + // file — bail out with InvalidInput rather than silently + // writing to a wrong place. + None => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "atomic_write_json: target path has no file name", + )); + } + }; + let payload = serde_json::to_vec(value) + .map_err(|e| std::io::Error::other(format!("serialize json: {e}")))?; + tokio::fs::write(&tmp_path, &payload).await?; + tokio::fs::rename(&tmp_path, final_path).await +} + /// Absolute on-disk location of a case directory. /// /// Single source of truth for the layout `///`. @@ -86,16 +120,12 @@ pub async fn read_oneliner_state( (Some(state), mtime) } -/// Write `state` atomically to `case_dir/oneliner.json` via -/// `write(tmp) + rename(tmp, final)`. Rename is atomic within a single -/// filesystem, so concurrent readers never see a half-serialized JSON -/// document. +/// Write `state` atomically to `case_dir/oneliner.json`. Thin wrapper +/// over [`atomic_write_json`] that fixes the filename — kept as its own +/// function so call-sites read as "write the oneliner state" rather +/// than threading the filename constant in by hand. pub async fn write_oneliner_state(case_dir: &Path, state: &OnelinerState) -> std::io::Result<()> { - let final_path = case_dir.join(ONELINER_FILENAME); - let tmp_path = case_dir.join(format!("{ONELINER_FILENAME}.tmp")); - let payload = serde_json::to_vec(state).expect("OnelinerState is infallibly serializable"); - tokio::fs::write(&tmp_path, &payload).await?; - tokio::fs::rename(&tmp_path, &final_path).await + atomic_write_json(&case_dir.join(ONELINER_FILENAME), state).await } /// Delete `case_dir/oneliner.json` unless it persists a manual override. @@ -138,21 +168,86 @@ pub async fn delete_oneliner_unless_manual( } } -/// Load the `.transcript.txt` sidecar for a recording as a +/// Load the `.json` metadata sidecar for a recording as a /// [`TranscriptState`]. /// /// `audio_stem_path` is the non-`.failed` stem path of the recording /// (e.g. `case_dir/2026-04-16T16-23-38Z.m4a`) — the sidecar sits next -/// to it with extension `.transcript.txt`. +/// to it with the `.json` extension. /// -/// This is the single reader for that sidecar. Every call-site that -/// used to do `tokio::fs::read_to_string(...).ok()` or +/// This is the single reader of that sidecar's transcript field. +/// Every call-site that used to do +/// `tokio::fs::read_to_string(...).ok()` or /// `with_extension("transcript.txt").exists()` should go through here /// so the three-state semantics stay consistent across the codebase. pub async fn read_transcript_state(audio_stem_path: &Path) -> TranscriptState { - let transcript_path = audio_stem_path.with_extension("transcript.txt"); - let raw = tokio::fs::read_to_string(&transcript_path).await.ok(); - TranscriptState::from_raw(raw.as_deref()) + TranscriptState::from_meta(read_recording_meta(audio_stem_path).await.as_ref()) +} + +/// Path of the per-recording metadata sidecar next to `audio_stem_path`. +/// +/// `audio_stem_path` is the non-`.failed` stem path of the recording +/// (e.g. `case_dir/2026-04-16T16-23-38Z.m4a`); the sidecar lives at +/// `case_dir/2026-04-16T16-23-38Z.json`. +pub fn recording_meta_path(audio_stem_path: &Path) -> PathBuf { + audio_stem_path.with_extension(RECORDING_META_SUFFIX) +} + +/// Read and parse the `.json` sidecar for a recording. +/// +/// Returns `None` for missing files (the typical "transcriber hasn't +/// finished yet" case) and for malformed JSON (logged at warn — a +/// rewritten sidecar will overwrite the bad one on the next worker +/// pass). Callers translate `None` to [`TranscriptState::Pending`]. +pub async fn read_recording_meta(audio_stem_path: &Path) -> Option { + let path = recording_meta_path(audio_stem_path); + let bytes = tokio::fs::read(&path).await.ok()?; + match serde_json::from_slice(&bytes) { + Ok(meta) => Some(meta), + Err(e) => { + tracing::warn!( + path = %path.display(), + error = %e, + "malformed recording meta sidecar; treating as missing" + ); + None + } + } +} + +/// Atomically write `.json` for a recording. Thin wrapper around +/// [`atomic_write_json`] that fixes the filename so call-sites read as +/// "persist this recording's metadata" rather than threading the +/// suffix in by hand. +pub async fn write_recording_meta_atomic( + audio_stem_path: &Path, + meta: &RecordingMeta, +) -> std::io::Result<()> { + atomic_write_json(&recording_meta_path(audio_stem_path), meta).await +} + +/// Synchronous test fixture: write a `.json` directly with the +/// given transcript and duration. Mirrors what the worker would have +/// persisted at the end of its pipeline. Async helpers exist for +/// production paths; this one keeps unit-test setups terse and +/// callable from non-async contexts. +#[cfg(test)] +pub(crate) fn write_recording_meta_sync( + case_dir: &Path, + stem: &str, + transcript: doctate_common::Transcript, + duration_seconds: Option, +) { + let meta = doctate_common::RecordingMeta { + transcript, + duration_seconds, + }; + let bytes = serde_json::to_vec(&meta).expect("RecordingMeta serializes infallibly"); + std::fs::write( + case_dir.join(format!("{stem}.{RECORDING_META_SUFFIX}")), + bytes, + ) + .expect("write recording meta sidecar in test fixture"); } #[cfg(test)] diff --git a/server/src/routes/case_actions.rs b/server/src/routes/case_actions.rs index 6d40979..bfb078f 100644 --- a/server/src/routes/case_actions.rs +++ b/server/src/routes/case_actions.rs @@ -11,8 +11,8 @@ use time::format_description::well_known::Rfc3339; use tokio::io::AsyncWriteExt; use tracing::{info, warn}; -use doctate_common::TranscriptState; use doctate_common::timestamp::{filename_stem_to_recorded_at, now_rfc3339}; +use doctate_common::{RECORDING_META_SUFFIX, TranscriptState}; use crate::analyze::{ ANALYSIS_INPUT_FILE, AnalysisInput, AnalyzeJob, AnalyzeSender, DOCUMENT_FILE, RecordingInput, @@ -171,7 +171,7 @@ fn resolve_list_return_path(headers: &HeaderMap) -> String { /// Build an `AnalysisInput` for the given case directory. /// Collects all `.m4a` (excluding `.m4a.failed`), reads the matching -/// `.transcript.txt` for each, skips blank transcripts, and computes +/// `.json` for each, skips blank transcripts, and computes /// `last_recording_mtime` as the max mtime over ALL `.m4a` (including blank /// ones — a late blank addendum still counts as activity). pub(crate) async fn build_analysis_input(case_dir: &Path) -> Result { @@ -225,7 +225,7 @@ async fn read_recordings(m4as: &[(PathBuf, SystemTime)]) -> Result continue, - TranscriptState::Content(t) => t, + TranscriptState::Content { text } => text, }; let stem = m4a_path.file_stem().and_then(|s| s.to_str()).unwrap_or(""); @@ -322,12 +322,27 @@ pub(crate) async fn reset_case_artefacts( } delete_oneliner_unless_manual(case_dir, locks).await?; let mut entries = tokio::fs::read_dir(case_dir).await?; + let meta_suffix = format!(".{RECORDING_META_SUFFIX}"); while let Some(entry) = entries.next_entry().await? { let name = entry.file_name(); let Some(s) = name.to_str() else { continue }; - if s.ends_with(".transcript.txt") { - tokio::fs::remove_file(entry.path()).await?; - } else if s.ends_with(".m4a.failed") { + // Per-recording metadata sidecar: paired with `.m4a` or + // its `.failed` twin. The pair check guards case-level JSONs + // (preserved Manual oneliner, future siblings) from being + // misidentified as recording artefacts. + if let Some(stem) = s.strip_suffix(&meta_suffix) { + let paired = tokio::fs::try_exists(case_dir.join(format!("{stem}.m4a"))) + .await + .unwrap_or(false) + || tokio::fs::try_exists(case_dir.join(format!("{stem}.m4a.failed"))) + .await + .unwrap_or(false); + if paired { + tokio::fs::remove_file(entry.path()).await?; + } + continue; + } + if s.ends_with(".m4a.failed") { let new_name = &s[..s.len() - ".failed".len()]; tokio::fs::rename(entry.path(), case_dir.join(new_name)).await?; } @@ -601,13 +616,13 @@ pub async fn handle_delete_recording( .trim_end_matches(".failed") .trim_end_matches(".m4a"); - // Audio and its sidecars. NotFound on any of these is fine: the - // transcript / duration may never have been written (silence, failed - // job, or the file got deleted twice). - let targets: [PathBuf; 5] = [ + // Audio and its single metadata sidecar. NotFound on any of these + // is fine: the JSON sidecar may never have been written (the + // worker hadn't reached the atomic write yet, or the file got + // deleted twice). + let targets: [PathBuf; 4] = [ case_dir.join(&form.filename), - case_dir.join(format!("{stem}.transcript.txt")), - case_dir.join(format!("{stem}.duration.txt")), + case_dir.join(format!("{stem}.{RECORDING_META_SUFFIX}")), case_dir.join(DOCUMENT_FILE), case_dir.join(ANALYSIS_INPUT_FILE), ]; diff --git a/server/src/routes/upload.rs b/server/src/routes/upload.rs index f6d5b4e..fdc59a7 100644 --- a/server/src/routes/upload.rs +++ b/server/src/routes/upload.rs @@ -116,7 +116,7 @@ pub async fn handle_upload( ); // Enqueue transcription. Failure here is non-fatal: on restart the recovery - // scan will re-enqueue any .m4a without a .transcript.txt sibling. + // scan will re-enqueue any .m4a without a .json sibling. match transcribe_tx.try_send(TranscribeJob { audio_path: file_path.clone(), user_slug: user.slug.clone(), diff --git a/server/src/routes/web.rs b/server/src/routes/web.rs index 810de9a..e0b3928 100644 --- a/server/src/routes/web.rs +++ b/server/src/routes/web.rs @@ -10,7 +10,6 @@ use doctate_common::TranscriptState; use doctate_common::timestamp::filename_stem_to_recorded_at; use tokio::io::{AsyncReadExt, AsyncSeekExt}; use tokio::task::JoinSet; -use tracing::debug; use crate::config::Config; use crate::error::AppError; @@ -22,9 +21,11 @@ pub(crate) struct RecordingView { /// RFC3339 UTC timestamp derived from the filename stem. Empty only /// on malformed filenames, which should not occur in production. pub(crate) recorded_at_iso: String, - /// Container duration in whole seconds, read from the `.duration.txt` - /// sidecar. `None` means neither the worker nor the lazy-backfill path - /// produced one — the UI then falls back to browser `preload="metadata"`. + /// Container duration in whole seconds, read from the `.json` + /// sidecar's `duration_seconds` field. `None` means the worker + /// hasn't finished yet (no sidecar) and the in-flight lazy ffprobe + /// could not produce one either — the UI then falls back to + /// browser `preload="metadata"`. pub(crate) duration_seconds: Option, /// Three-way transcript state — `Pending` (not yet transcribed), /// `Silent` (transcriber classified as silence), or `Content(text)`. @@ -164,16 +165,15 @@ fn stem_of(filename: &str) -> &str { .trim_end_matches(".m4a") } -/// One entry in progress between the directory enumeration and the optional -/// parallel ffprobe backfill. Holds the resolved paths so the spawned task -/// doesn't have to know about the case_dir. +/// One entry in progress between the directory enumeration and the +/// in-flight ffprobe backfill. The audio path is kept around so the +/// spawned task can probe directly without re-deriving the location. struct RawRecording { filename: String, failed: bool, transcript: TranscriptState, duration_seconds: Option, audio_path: PathBuf, - sidecar_path: PathBuf, } pub(crate) async fn scan_recordings(case_dir: &Path) -> Vec { @@ -193,17 +193,14 @@ pub(crate) async fn scan_recordings(case_dir: &Path) -> Vec { if !is_audio { continue; } - // Transcript and duration sidecar both hang off the non-failed stem path; - // a `.failed` file never produced a transcript but the lookup still works. + // Transcript and duration both come from the per-recording + // metadata sidecar; a `.failed` file never produced one, but + // the lookup still works (returns None → Pending state). let stem_file = filename.trim_end_matches(".failed"); let stem_path = case_dir.join(stem_file); - let transcript = paths::read_transcript_state(&stem_path).await; - - let sidecar_path = stem_path.with_extension("duration.txt"); - let duration_seconds = tokio::fs::read_to_string(&sidecar_path) - .await - .ok() - .and_then(|s| s.trim().parse::().ok()); + let meta = paths::read_recording_meta(&stem_path).await; + let duration_seconds = meta.as_ref().and_then(|m| m.duration_seconds); + let transcript = TranscriptState::from_meta(meta.as_ref()); let audio_path = case_dir.join(&filename); raws.push(RawRecording { @@ -212,25 +209,25 @@ pub(crate) async fn scan_recordings(case_dir: &Path) -> Vec { transcript, duration_seconds, audio_path, - sidecar_path, }); } - // Lazy backfill: spawn ffprobe in parallel for recordings without a sidecar. - // `.failed` entries are skipped — ffprobe on a corrupt file is unlikely to - // succeed and the UI doesn't need a duration there. + // Read-only lazy backfill: while the transcribe worker is still in + // flight (no sidecar yet), run ffprobe in parallel for the UI render + // so the recordings list shows a duration during the transcription + // window. The result is *not* persisted — the worker is the single + // source of truth for `.json` and writes both transcript and + // duration in one atomic step at the end of its pipeline. + // `.failed` entries are skipped: ffprobe on a corrupt file is + // unlikely to succeed and the UI doesn't need a duration there. let mut js: JoinSet> = JoinSet::new(); for (idx, r) in raws.iter().enumerate() { if r.duration_seconds.is_some() || r.failed { continue; } let audio_path = r.audio_path.clone(); - let sidecar_path = r.sidecar_path.clone(); js.spawn(async move { let secs = ffmpeg::probe_duration_seconds(&audio_path).await.ok()?; - if let Err(e) = tokio::fs::write(&sidecar_path, secs.to_string()).await { - debug!(sidecar = %sidecar_path.display(), error = %e, "duration sidecar write failed"); - } Some((idx, secs)) }); } @@ -266,20 +263,19 @@ mod tests { use tempfile::tempdir; #[tokio::test] - async fn scan_reads_duration_sidecar_and_derives_iso() { + async fn scan_reads_meta_sidecar_and_derives_iso() { let dir = tempdir().unwrap(); tokio::fs::write(dir.path().join("2026-04-19T10-00-00Z.m4a"), b"audio") .await .unwrap(); - tokio::fs::write(dir.path().join("2026-04-19T10-00-00Z.duration.txt"), "42") - .await - .unwrap(); - tokio::fs::write( - dir.path().join("2026-04-19T10-00-00Z.transcript.txt"), - "hallo", - ) - .await - .unwrap(); + crate::paths::write_recording_meta_sync( + dir.path(), + "2026-04-19T10-00-00Z", + doctate_common::Transcript::Content { + text: "hallo".into(), + }, + Some(42), + ); let got = scan_recordings(dir.path()).await; assert_eq!(got.len(), 1); @@ -335,21 +331,22 @@ mod tests { } #[tokio::test] - async fn scan_skips_corrupt_sidecar() { + async fn scan_handles_malformed_meta_sidecar() { let dir = tempdir().unwrap(); tokio::fs::write(dir.path().join("2026-04-19T12-00-00Z.m4a"), b"audio") .await .unwrap(); - tokio::fs::write( - dir.path().join("2026-04-19T12-00-00Z.duration.txt"), - "not-a-number", - ) - .await - .unwrap(); - // No ffprobe available in unit-test context (or it fails on "audio"), so - // duration_seconds remains None — exactly what we want to assert. + // Garbage JSON: read_recording_meta logs a warning and returns + // None, so the recording surfaces as Pending with no duration. + // The lazy ffprobe backfill would normally produce a duration, + // but ffprobe on the placeholder bytes fails — leaving None, + // exactly what we want to assert. + tokio::fs::write(dir.path().join("2026-04-19T12-00-00Z.json"), "not-a-json") + .await + .unwrap(); let got = scan_recordings(dir.path()).await; assert_eq!(got.len(), 1); assert!(got[0].duration_seconds.is_none()); + assert!(got[0].transcript.is_pending()); } } diff --git a/server/src/transcribe/recovery.rs b/server/src/transcribe/recovery.rs index 5c39742..cfff212 100644 --- a/server/src/transcribe/recovery.rs +++ b/server/src/transcribe/recovery.rs @@ -1,5 +1,6 @@ use std::path::{Path, PathBuf}; +use doctate_common::RECORDING_META_SUFFIX; use doctate_common::oneliners::OnelinerState; use tracing::{info, warn}; @@ -35,8 +36,9 @@ pub async fn scan_and_enqueue(data_path: &Path, tx: &TranscribeSender) { } /// Scan a single `//*.m4a` set and enqueue every recording -/// whose `.transcript.txt` is missing. Returns the number sent. Channel send -/// is awaited (bounded channel); only fails if the worker is gone. +/// whose metadata sidecar (`.json`) is missing. Returns the number +/// sent. Channel send is awaited (bounded channel); only fails if the +/// worker is gone. pub async fn enqueue_pending_for_user( user_root: &Path, slug: &str, @@ -165,13 +167,17 @@ pub(crate) async fn cases_needing_oneliner_in(user_root: &Path) -> Vec /// meaning the transcribe pipeline will produce no further artefacts /// and `update_oneliner` has enough information to settle the case. /// -/// Three terminal shapes count: -/// - `.transcript.txt` with content → [`TranscriptState::Content`] -/// - `.transcript.txt` empty → [`TranscriptState::Silent`] -/// - `.m4a.failed` → permanent transcription failure (ffmpeg remux or -/// Whisper 4xx). Transient Whisper errors leave the file as plain -/// `.m4a` without sidecar (Pending), so they do *not* match here — -/// the page-load heal re-enqueues them on the next refresh. +/// Two terminal shapes count: +/// - `.json` next to `.m4a` → transcriber wrote its +/// metadata sidecar (carries either Silent or Content). The +/// `.m4a` pairing is required because the case directory also +/// holds case-level JSONs (`oneliner.json`, `analysis_input.json`) +/// that must not be misread as a recording sidecar. +/// - `.m4a.failed` → permanent transcription failure (ffmpeg +/// remux or Whisper 4xx). Transient Whisper errors leave the file +/// as plain `.m4a` without sidecar (Pending), so they do *not* +/// match here — the page-load heal re-enqueues them on the next +/// refresh. /// /// Cases where every recording is still `Pending` (plain `.m4a`, no /// sidecar, not marked failed) return false: the transcriber hasn't @@ -186,9 +192,19 @@ async fn has_any_transcript(case_dir: &Path) -> bool { return false; }; while let Ok(Some(f)) = files.next_entry().await { - if f.file_name() - .to_str() - .is_some_and(|s| s.ends_with(".transcript.txt") || s.ends_with(".m4a.failed")) + let Some(name) = f.file_name().to_str().map(str::to_owned) else { + continue; + }; + if name.ends_with(".m4a.failed") { + return true; + } + // A per-recording meta sidecar's stem matches its audio's + // stem. Pair the JSON with `.m4a` to filter out + // case-level JSONs (oneliner.json, analysis_input.json). + if let Some(stem) = name.strip_suffix(&format!(".{RECORDING_META_SUFFIX}")) + && tokio::fs::try_exists(case_dir.join(format!("{stem}.m4a"))) + .await + .unwrap_or(false) { return true; } @@ -246,16 +262,33 @@ pub async fn regenerate_missing_oneliners_for_user( #[cfg(test)] mod tests { use super::*; + use doctate_common::Transcript; use tempfile::tempdir; + /// Seed `.m4a` plus its `.json` metadata sidecar — the + /// shape `has_any_transcript` recognises as terminal. The pair + /// (audio + sidecar) is required because recovery filters + /// case-level JSONs from the per-recording set by checking for an + /// audio sibling. + async fn seed_recording(case: &Path, stem: &str, transcript: Transcript) { + tokio::fs::write(case.join(format!("{stem}.m4a")), b"audio") + .await + .unwrap(); + crate::paths::write_recording_meta_sync(case, stem, transcript, None); + } + + fn content(text: &str) -> Transcript { + Transcript::Content { + text: text.to_owned(), + } + } + #[tokio::test] async fn cases_needing_oneliner_finds_crash_survivor() { let data = tempdir().unwrap(); let case = data.path().join("user").join("case1"); tokio::fs::create_dir_all(&case).await.unwrap(); - tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), "content") - .await - .unwrap(); + seed_recording(&case, "2026-04-16T10-00-00Z", content("content")).await; let got = cases_needing_oneliner(data.path()).await; assert_eq!(got, vec![(case, "user".to_owned())]); @@ -273,9 +306,7 @@ mod tests { let data = tempdir().unwrap(); let case = data.path().join("user").join("case1"); tokio::fs::create_dir_all(&case).await.unwrap(); - tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), "content") - .await - .unwrap(); + seed_recording(&case, "2026-04-16T10-00-00Z", content("content")).await; write_state( &case, &OnelinerState::Ready { @@ -293,9 +324,7 @@ mod tests { let data = tempdir().unwrap(); let case = data.path().join("user").join("case1"); tokio::fs::create_dir_all(&case).await.unwrap(); - tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), "content") - .await - .unwrap(); + seed_recording(&case, "2026-04-16T10-00-00Z", content("content")).await; write_state( &case, &OnelinerState::Empty { @@ -312,9 +341,7 @@ mod tests { let data = tempdir().unwrap(); let case = data.path().join("user").join("case1"); tokio::fs::create_dir_all(&case).await.unwrap(); - tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), "content") - .await - .unwrap(); + seed_recording(&case, "2026-04-16T10-00-00Z", content("content")).await; write_state( &case, &OnelinerState::Error { @@ -327,27 +354,25 @@ mod tests { assert_eq!(got, vec![(case, "user".to_owned())]); } - /// Silent-only cases (transcript file exists but is empty/whitespace) - /// are now picked up by recovery so `update_oneliner` can settle - /// them to [`OnelinerState::Empty`]. Previously they were skipped, - /// which left the UI stuck on "Generating" forever (the bug this - /// enum refactor fixes). + /// Silent-only cases (sidecar carries `Transcript::Silent`) are + /// picked up by recovery so `update_oneliner` can settle them to + /// [`OnelinerState::Empty`]. Previously they were skipped, which + /// left the UI stuck on "Generating" forever (the bug the + /// three-state model fixes). #[tokio::test] async fn cases_needing_oneliner_picks_up_silent_only_cases_for_empty_settlement() { let data = tempdir().unwrap(); let case = data.path().join("user").join("case1"); tokio::fs::create_dir_all(&case).await.unwrap(); - tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), " ") - .await - .unwrap(); + seed_recording(&case, "2026-04-16T10-00-00Z", Transcript::Silent).await; let got = cases_needing_oneliner(data.path()).await; assert_eq!(got, vec![(case, "user".to_owned())]); } - /// Fully-pending cases (no transcript file yet) are skipped — + /// Fully-pending cases (no sidecar yet) are skipped — /// `update_oneliner` has nothing to act on until the transcriber - /// writes a sidecar. + /// writes one. #[tokio::test] async fn cases_needing_oneliner_skips_fully_pending_cases() { let data = tempdir().unwrap(); @@ -356,7 +381,7 @@ mod tests { tokio::fs::write(case.join("2026-04-16T10-00-00Z.m4a"), b"audio") .await .unwrap(); - // No .transcript.txt — the recording is still Pending. + // No .json — the recording is still Pending. assert!(cases_needing_oneliner(data.path()).await.is_empty()); } @@ -366,9 +391,7 @@ mod tests { let data = tempdir().unwrap(); let case = data.path().join("user").join("case1"); tokio::fs::create_dir_all(&case).await.unwrap(); - tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), "content") - .await - .unwrap(); + seed_recording(&case, "2026-04-16T10-00-00Z", content("content")).await; // Empty JSON object is treated as a valid close marker // (see paths::is_closed — existence of the file is enough). tokio::fs::write(case.join(".closed"), "{}").await.unwrap(); diff --git a/server/src/transcribe/worker.rs b/server/src/transcribe/worker.rs index ce533ab..af78498 100644 --- a/server/src/transcribe/worker.rs +++ b/server/src/transcribe/worker.rs @@ -2,8 +2,8 @@ use std::path::Path; use std::sync::Arc; use std::time::Duration; -use doctate_common::TranscriptState; use doctate_common::oneliners::OnelinerState; +use doctate_common::{RecordingMeta, Transcript, TranscriptState}; use time::OffsetDateTime; use time::format_description::well_known::Rfc3339; use tracing::{error, info, warn}; @@ -38,9 +38,13 @@ pub async fn run( while let Some(job) = rx.recv().await { let _guard = BusyGuard::new(worker_busy.clone()); let audio_path = job.audio_path; - let transcript_path = audio_path.with_extension("transcript.txt"); + let meta_path = paths::recording_meta_path(&audio_path); - if transcript_path.exists() { + // Idempotency gate: presence of `.json` means the + // transcriber already finished this recording (single atomic + // write at the end of the pipeline). Skip the job, freeing + // Whisper for the next one. + if meta_path.exists() { continue; } @@ -65,11 +69,24 @@ pub async fn run( } }; - // Persist the container duration as a sidecar so the UI can render it - // without the browser having to HEAD every audio file. Non-fatal — the - // recordings scan has a lazy-backfill path, and transcription is the - // critical workload here. - write_duration_sidecar(&audio_path, remuxed.path()).await; + // Probe the container duration on the remuxed copy (its `moov` + // atom sits at the start, so ffprobe returns without seeking + // to EOF). We carry the value in memory and persist it + // together with the transcript at the end of the pipeline. + // Non-fatal: a probe failure leaves `duration_seconds = None` + // and the recordings scan falls back to a read-only ffprobe + // for the UI render. + let duration_seconds = match ffmpeg::probe_duration_seconds(remuxed.path()).await { + Ok(s) => Some(s), + Err(e) => { + warn!( + audio = %audio_path.display(), + error = %e, + "ffprobe duration failed — meta will record null duration" + ); + None + } + }; let text = match whisper::transcribe( &client, @@ -106,15 +123,31 @@ pub async fn run( // the canonical path. let text = vocab.replace(&text); - if let Err(e) = tokio::fs::write(&transcript_path, &text).await { - error!(transcript = %transcript_path.display(), error = %e, "writing transcript failed"); + // Single atomic write: assemble the recording metadata + // (transcript outcome + duration) in memory, then materialise + // it on disk in one rename. Either readers see the previous + // state (no sidecar = pending) or the full new sidecar — never + // a half-written one. + let bytes_logged = text.len(); + let transcript = if text.trim().is_empty() { + Transcript::Silent + } else { + Transcript::Content { text } + }; + let meta = RecordingMeta { + transcript, + duration_seconds, + }; + if let Err(e) = paths::write_recording_meta_atomic(&audio_path, &meta).await { + error!(meta = %meta_path.display(), error = %e, "writing recording meta failed"); continue; } info!( audio = %audio_path.display(), - bytes = text.len(), - "Transcript written" + bytes = bytes_logged, + duration = ?duration_seconds, + "Recording meta written" ); // Notify the live-update bus so subscribed browsers can refresh the @@ -185,23 +218,6 @@ async fn mark_failed(audio_path: &Path, events_tx: &EventSender, user_slug: &str } } -/// Probe the (remuxed) audio and persist the duration next to the original file -/// as `.duration.txt`. Probes on the remuxed copy because its `moov` atom -/// sits at the start — ffprobe returns without seeking to EOF. -async fn write_duration_sidecar(audio_path: &Path, probe_path: &Path) { - let secs = match ffmpeg::probe_duration_seconds(probe_path).await { - Ok(s) => s, - Err(e) => { - warn!(audio = %audio_path.display(), error = %e, "ffprobe duration failed — UI will lazy-backfill"); - return; - } - }; - let sidecar = audio_path.with_extension("duration.txt"); - if let Err(e) = tokio::fs::write(&sidecar, secs.to_string()).await { - warn!(sidecar = %sidecar.display(), error = %e, "writing duration sidecar failed"); - } -} - /// Regenerate `case_dir/oneliner.json` from **all** `Content` transcripts /// in the case, joined chronologically. Called after every successful /// transcript write so later recordings can correct earlier ones (mirror @@ -408,7 +424,7 @@ pub(crate) async fn all_transcripts_joined(case_dir: &Path) -> Option { let mut parts: Vec = Vec::with_capacity(m4as.len()); for path in m4as { - if let TranscriptState::Content(text) = paths::read_transcript_state(&path).await { + if let TranscriptState::Content { text } = paths::read_transcript_state(&path).await { let trimmed = text.trim(); if !trimmed.is_empty() { parts.push(trimmed.to_string()); @@ -426,18 +442,27 @@ mod tests { use super::*; use tempfile::tempdir; - /// Helper: seed an m4a plus an optional transcript sidecar, matching + /// Helper: seed an m4a plus an optional metadata sidecar, matching /// how the worker lays files out on disk. `all_transcripts_joined` /// iterates `.m4a` files and reads the sidecar through the shared - /// reader, so bare transcripts without m4as would be ignored. + /// reader, so bare meta files without m4as would be ignored. + /// + /// `transcript`: `None` → no sidecar (Pending); `Some(s)` with + /// whitespace-only `s` → `Transcript::Silent`; otherwise + /// `Transcript::Content`. async fn seed(dir: &Path, stem: &str, transcript: Option<&str>) { tokio::fs::write(dir.join(format!("{stem}.m4a")), b"audio") .await .unwrap(); if let Some(text) = transcript { - tokio::fs::write(dir.join(format!("{stem}.transcript.txt")), text) - .await - .unwrap(); + let variant = if text.trim().is_empty() { + Transcript::Silent + } else { + Transcript::Content { + text: text.to_owned(), + } + }; + paths::write_recording_meta_sync(dir, stem, variant, None); } } @@ -503,15 +528,7 @@ mod tests { #[tokio::test] async fn has_pending_recordings_true_when_m4a_without_transcript() { let dir = tempdir().unwrap(); - tokio::fs::write(dir.path().join("2026-04-16T10-00-00Z.m4a"), b"a") - .await - .unwrap(); - tokio::fs::write( - dir.path().join("2026-04-16T10-00-00Z.transcript.txt"), - "text", - ) - .await - .unwrap(); + seed(dir.path(), "2026-04-16T10-00-00Z", Some("text")).await; tokio::fs::write(dir.path().join("2026-04-16T10-05-00Z.m4a"), b"b") .await .unwrap(); @@ -521,24 +538,8 @@ mod tests { #[tokio::test] async fn has_pending_recordings_false_when_all_transcribed() { let dir = tempdir().unwrap(); - tokio::fs::write(dir.path().join("2026-04-16T10-00-00Z.m4a"), b"a") - .await - .unwrap(); - tokio::fs::write( - dir.path().join("2026-04-16T10-00-00Z.transcript.txt"), - "text1", - ) - .await - .unwrap(); - tokio::fs::write(dir.path().join("2026-04-16T10-05-00Z.m4a"), b"b") - .await - .unwrap(); - tokio::fs::write( - dir.path().join("2026-04-16T10-05-00Z.transcript.txt"), - "text2", - ) - .await - .unwrap(); + seed(dir.path(), "2026-04-16T10-00-00Z", Some("text1")).await; + seed(dir.path(), "2026-04-16T10-05-00Z", Some("text2")).await; assert!(!has_pending_recordings(dir.path()).await); } diff --git a/server/templates/case_recordings.html b/server/templates/case_recordings.html index 92474a2..061005d 100644 --- a/server/templates/case_recordings.html +++ b/server/templates/case_recordings.html @@ -164,8 +164,8 @@ try {
Transkription fehlgeschlagen
{% else %} {% match rec.transcript %} -{% when TranscriptState::Content with (t) %} -
{{ t }}
+{% when TranscriptState::Content with { text } %} +
{{ text }}
{% when TranscriptState::Silent %}
(stille Aufnahme)
{% when TranscriptState::Pending %} diff --git a/server/tests/analyze_test.rs b/server/tests/analyze_test.rs index e58be73..cf6581c 100644 --- a/server/tests/analyze_test.rs +++ b/server/tests/analyze_test.rs @@ -1205,7 +1205,7 @@ async fn reset_case_clears_derived_and_unfails_audio() { assert!(dir.join("2026-04-15T09-05-00Z.m4a").exists()); assert!(!dir.join("2026-04-15T09-05-00Z.m4a.failed").exists()); // Derived artefacts gone. - assert!(!dir.join("2026-04-15T09-00-00Z.transcript.txt").exists()); + assert!(!dir.join("2026-04-15T09-00-00Z.json").exists()); assert!(!dir.join(ONELINER_FILENAME).exists()); assert!(!dir.join(DOCUMENT_FILE).exists()); assert!(!dir.join(ANALYSIS_INPUT_FILE).exists()); @@ -1235,7 +1235,7 @@ async fn reset_case_requires_admin() { assert_eq!(resp.status(), StatusCode::FORBIDDEN); // Nothing touched. - assert!(dir.join("2026-04-15T09-00-00Z.transcript.txt").exists()); + assert!(dir.join("2026-04-15T09-00-00Z.json").exists()); assert!(dir.join(DOCUMENT_FILE).exists()); } @@ -1282,8 +1282,8 @@ async fn bulk_reset_processes_multiple_cases_admin_only() { assert_eq!(resp.status(), StatusCode::SEE_OTHER); assert!(!dir_a.join(DOCUMENT_FILE).exists()); assert!(!dir_b.join(DOCUMENT_FILE).exists()); - assert!(!dir_a.join("2026-04-15T10-00-00Z.transcript.txt").exists()); - assert!(!dir_b.join("2026-04-15T10-05-00Z.transcript.txt").exists()); + assert!(!dir_a.join("2026-04-15T10-00-00Z.json").exists()); + assert!(!dir_b.join("2026-04-15T10-05-00Z.json").exists()); // Non-admin: bulk reset is rejected, dr_b's case untouched. let (cookie_doctor, csrf_doctor) = login_with_csrf(&app, &store, "dr_b", "s").await; @@ -1299,7 +1299,7 @@ async fn bulk_reset_processes_multiple_cases_admin_only() { .unwrap(); assert_eq!(resp.status(), StatusCode::FORBIDDEN); assert!(dir_c.join(DOCUMENT_FILE).exists()); - assert!(dir_c.join("2026-04-15T11-00-00Z.transcript.txt").exists()); + assert!(dir_c.join("2026-04-15T11-00-00Z.json").exists()); } /// Regression guard: the bulk route handles *all* actions (including diff --git a/server/tests/common/mod.rs b/server/tests/common/mod.rs index d8a1d69..f051fe5 100644 --- a/server/tests/common/mod.rs +++ b/server/tests/common/mod.rs @@ -27,13 +27,15 @@ pub mod users; // Re-exports for ergonomic `use common::*;` in most test files. pub use artefacts::{ANALYSIS_INPUT_FILE, CLOSE_MARKER, DOCUMENT_FILE, ONELINER_FILENAME}; pub use config::{TestConfig, unique_tmpdir}; +pub use doctate_common::Transcript; pub use http::{ body_bytes, body_json, body_string, csrf_form_post, form_post, get_with_api_key, get_with_api_key_if_none_match, get_with_cookie, header_opt, header_str, multipart_upload_body, }; pub use seed::{ past_rfc3339, seed_case, seed_oneliner_ready, seed_oneliner_state, seed_recording, - seed_recording_with_age, seed_recording_with_sidecars, write_closed_marker, + seed_recording_meta, seed_recording_with_age, seed_recording_with_sidecars, + write_closed_marker, }; pub use session::{extract_session_cookie, login, login_request, login_with_csrf}; pub use users::{ diff --git a/server/tests/common/seed.rs b/server/tests/common/seed.rs index ed4fd1a..91ae976 100644 --- a/server/tests/common/seed.rs +++ b/server/tests/common/seed.rs @@ -11,8 +11,8 @@ use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; -use doctate_common::ONELINER_FILENAME; use doctate_common::oneliners::OnelinerState; +use doctate_common::{ONELINER_FILENAME, RECORDING_META_SUFFIX, RecordingMeta, Transcript}; use doctate_server::paths::CLOSE_MARKER; use filetime::FileTime; @@ -26,28 +26,71 @@ pub fn seed_case(data_path: &Path, slug: &str, case_id: &str) -> PathBuf { } /// Write `.m4a` with placeholder bytes, plus optionally a -/// `.transcript.txt`. `stem` must be the full RFC3339-like -/// filename stem (e.g. `2026-04-15T10-00-00Z`); pass the same prefix -/// you want to read back later. Returns the audio filename (no path) -/// so the caller can feed it back to the delete endpoint. +/// `.json` sidecar that mirrors the transcribe outcome. +/// +/// `transcript`: +/// - `None` → no sidecar written (recording stays in `Pending` state). +/// - `Some(text)` with whitespace-only `text` → sidecar with +/// `Transcript::Silent` (matches the old `transcript.txt = ""` rule). +/// - `Some(text)` non-empty → sidecar with `Transcript::Content`. +/// +/// Returns the audio filename (no path) so the caller can feed it back +/// to the delete endpoint. pub fn seed_recording(case_dir: &Path, stem: &str, transcript: Option<&str>) -> String { let filename = format!("{stem}.m4a"); std::fs::write(case_dir.join(&filename), b"audio-bytes").unwrap(); if let Some(text) = transcript { - std::fs::write(case_dir.join(format!("{stem}.transcript.txt")), text).unwrap(); + seed_recording_meta(case_dir, stem, transcript_for(text), None); } filename } -/// Like [`seed_recording`] but additionally writes the `.duration.txt` -/// sidecar — used by the delete-recording tests that assert every -/// sidecar is cleaned up. +/// Like [`seed_recording`] but the sidecar carries a real duration +/// (`duration_seconds: Some(42)`). Used by tests that assert the +/// duration is rendered or cleaned up. pub fn seed_recording_with_sidecars(case_dir: &Path, stem: &str, transcript: &str) -> String { - let filename = seed_recording(case_dir, stem, Some(transcript)); - std::fs::write(case_dir.join(format!("{stem}.duration.txt")), "42").unwrap(); + let filename = format!("{stem}.m4a"); + std::fs::write(case_dir.join(&filename), b"audio-bytes").unwrap(); + seed_recording_meta(case_dir, stem, transcript_for(transcript), Some(42)); filename } +/// Write a fully-specified `.json` next to an already-seeded +/// `.m4a`. Use when a test needs precise control over the +/// transcript variant or duration; the higher-level helpers above +/// cover the common cases. +pub fn seed_recording_meta( + case_dir: &Path, + stem: &str, + transcript: Transcript, + duration_seconds: Option, +) { + let meta = RecordingMeta { + transcript, + duration_seconds, + }; + let bytes = serde_json::to_vec(&meta).unwrap(); + std::fs::write( + case_dir.join(format!("{stem}.{RECORDING_META_SUFFIX}")), + bytes, + ) + .unwrap(); +} + +/// Map a free-text transcript to the `Transcript` variant the worker +/// would have produced for it: whitespace-only → `Silent`, otherwise +/// `Content`. Mirrors the old `transcript.txt = "" ⇒ Silent` rule so +/// existing tests keep their meaning. +fn transcript_for(text: &str) -> Transcript { + if text.trim().is_empty() { + Transcript::Silent + } else { + Transcript::Content { + text: text.to_owned(), + } + } +} + /// Write an m4a with an artificial `mtime` in the past (relative to /// now). Used by the retention-sweep tests to simulate "recorded N days /// ago" without a clock abstraction. diff --git a/server/tests/delete_recording_test.rs b/server/tests/delete_recording_test.rs index 75de2be..7519d50 100644 --- a/server/tests/delete_recording_test.rs +++ b/server/tests/delete_recording_test.rs @@ -1,7 +1,7 @@ //! Per-recording delete endpoint: POST /web/cases/{case_id}/recordings/delete. //! //! Verifies the endpoint: -//! - removes the audio file and its transcript / duration sidecars +//! - removes the audio file and its `.json` metadata sidecar //! - invalidates derived artefacts (oneliner.json, document.md, analysis_input.json) //! - leaves other recordings in the same case untouched //! - rejects path-traversal filenames with 400 @@ -68,26 +68,18 @@ async fn delete_recording_removes_file_and_sidecars_and_invalidates_derived() { "delete should redirect back to the recording list, not the case list" ); - // Victim + its sidecars are gone. + // Victim + its sidecar are gone. assert!(!case_dir.join(&victim).exists(), "m4a should be deleted"); assert!( - !case_dir - .join("2026-04-19T10-00-00Z.transcript.txt") - .exists(), - "transcript sidecar should be deleted" - ); - assert!( - !case_dir.join("2026-04-19T10-00-00Z.duration.txt").exists(), - "duration sidecar should be deleted" + !case_dir.join("2026-04-19T10-00-00Z.json").exists(), + "metadata sidecar should be deleted" ); // Survivor is intact. assert!(case_dir.join(&survivor).exists(), "other m4a must remain"); assert!( - case_dir - .join("2026-04-19T11-00-00Z.transcript.txt") - .exists(), - "other transcript must remain" + case_dir.join("2026-04-19T11-00-00Z.json").exists(), + "other metadata sidecar must remain" ); // Derived artefacts are invalidated. diff --git a/server/tests/oneliner_heal_decoupled_test.rs b/server/tests/oneliner_heal_decoupled_test.rs index 7fb7803..2816ce5 100644 --- a/server/tests/oneliner_heal_decoupled_test.rs +++ b/server/tests/oneliner_heal_decoupled_test.rs @@ -26,17 +26,20 @@ use wiremock::{Mock, MockServer, ResponseTemplate}; use common::{ONELINER_FILENAME, TestConfig}; fn write_transcript(case_dir: &Path) { - // Seed an m4a + transcript pair, matching the worker's on-disk - // layout. The oneliner worker iterates `.m4a` files and resolves - // the sidecar from the stem; a stray transcript without an m4a - // would be invisible to it. + // Seed an m4a + metadata sidecar pair, matching the worker's + // on-disk layout. The oneliner worker iterates `.m4a` files and + // resolves the sidecar from the stem; a stray sidecar without an + // m4a would be invisible to it. let stem = "2026-04-16T10-00-00Z"; std::fs::write(case_dir.join(format!("{stem}.m4a")), b"audio").unwrap(); - std::fs::write( - case_dir.join(format!("{stem}.transcript.txt")), - "Brustschmerz links, seit heute morgen.", - ) - .unwrap(); + common::seed_recording_meta( + case_dir, + stem, + common::Transcript::Content { + text: "Brustschmerz links, seit heute morgen.".to_owned(), + }, + None, + ); } #[tokio::test] diff --git a/server/tests/oneliner_override_test.rs b/server/tests/oneliner_override_test.rs index 1cf63ec..54165ae 100644 --- a/server/tests/oneliner_override_test.rs +++ b/server/tests/oneliner_override_test.rs @@ -250,11 +250,14 @@ async fn early_latch_skips_llm_call_when_manual_set() { // Also seed a transcript so update_oneliner has content to pass to LLM // (would call without latch). std::fs::write(case_dir.join("2026-04-26T11-00-00Z.m4a"), b"audio-bytes").unwrap(); - std::fs::write( - case_dir.join("2026-04-26T11-00-00Z.transcript.txt"), - "Patient hat Fieber.", - ) - .unwrap(); + common::seed_recording_meta( + &case_dir, + "2026-04-26T11-00-00Z", + common::Transcript::Content { + text: "Patient hat Fieber.".into(), + }, + None, + ); let locks = OnelinerLocks::new(); let events_tx = doctate_server::events::channel(); @@ -316,11 +319,14 @@ async fn re_check_drops_llm_result_when_manual_appears_during_regen() { // Seed transcript so update_oneliner enters the LLM branch. std::fs::write(case_dir.join("2026-04-26T11-00-00Z.m4a"), b"audio-bytes").unwrap(); - std::fs::write( - case_dir.join("2026-04-26T11-00-00Z.transcript.txt"), - "Patient klagt über Schwindel.", - ) - .unwrap(); + common::seed_recording_meta( + &case_dir, + "2026-04-26T11-00-00Z", + common::Transcript::Content { + text: "Patient klagt über Schwindel.".into(), + }, + None, + ); let locks = OnelinerLocks::new(); let events_tx = doctate_server::events::channel(); diff --git a/server/tests/silent_case_empty_test.rs b/server/tests/silent_case_empty_test.rs index b3c31fd..fcdbc13 100644 --- a/server/tests/silent_case_empty_test.rs +++ b/server/tests/silent_case_empty_test.rs @@ -28,11 +28,11 @@ use wiremock::{Mock, MockServer, ResponseTemplate}; use common::{ONELINER_FILENAME, TestConfig}; -/// Seed an m4a plus a silent (0-byte) transcript sidecar — the on-disk -/// shape of a recording that whisper classified as silence. +/// Seed an m4a plus a silent metadata sidecar — the on-disk shape of +/// a recording that whisper classified as silence. fn seed_silent_recording(case_dir: &Path, stem: &str) { std::fs::write(case_dir.join(format!("{stem}.m4a")), b"audio").unwrap(); - std::fs::write(case_dir.join(format!("{stem}.transcript.txt")), b"").unwrap(); + common::seed_recording_meta(case_dir, stem, common::Transcript::Silent, None); } #[tokio::test] diff --git a/server/tests/transcribe_test.rs b/server/tests/transcribe_test.rs index 78fa411..9976e0c 100644 --- a/server/tests/transcribe_test.rs +++ b/server/tests/transcribe_test.rs @@ -236,7 +236,14 @@ async fn recovery_enqueues_only_pending_recordings() { std::fs::create_dir_all(&case_a).unwrap(); std::fs::write(case_a.join("2026-04-10T10-00-00Z.m4a"), b"x").unwrap(); std::fs::write(case_a.join("2026-04-11T10-00-00Z.m4a"), b"x").unwrap(); - std::fs::write(case_a.join("2026-04-11T10-00-00Z.transcript.txt"), b"done").unwrap(); + common::seed_recording_meta( + &case_a, + "2026-04-11T10-00-00Z", + common::Transcript::Content { + text: "done".into(), + }, + None, + ); // User 2, case with one pending .m4a. let case_b = root.join("dr_b/bbbb"); @@ -380,14 +387,14 @@ async fn worker_leaves_m4a_intact_on_transient_whisper_error() { "transient error must not mark {} as failed", failed.display() ); - // And no transcript yet either (the 503 produced no text). - let transcript = case_dir.join("2026-04-13T10-31-00Z.transcript.txt"); - assert!(!transcript.exists(), "no transcript expected on 503"); + // And no metadata sidecar yet either (the 503 produced no text). + let meta = case_dir.join("2026-04-13T10-31-00Z.json"); + assert!(!meta.exists(), "no recording meta expected on 503"); } /// The worker must route the Whisper response through the Gazetteer /// before persisting. Mock returns the drift form "Zerebrum"; the -/// persisted `.transcript.txt` must contain the canonical "Cerebrum". +/// persisted `.json` must carry the canonical "Cerebrum". #[tokio::test] async fn transcribe_worker_normalizes_whisper_output() { let server = MockServer::start().await; @@ -438,10 +445,16 @@ async fn transcribe_worker_normalizes_whisper_output() { ) .await; - let transcript_path = case_dir.join("2026-04-13T10-30-00Z.transcript.txt"); - assert!(transcript_path.exists(), "transcript missing"); - let transcript = std::fs::read_to_string(&transcript_path).unwrap(); - assert_eq!(transcript, "Patient mit Blutung im Cerebrum."); + let meta_path = case_dir.join("2026-04-13T10-30-00Z.json"); + assert!(meta_path.exists(), "recording meta missing"); + let bytes = std::fs::read(&meta_path).unwrap(); + let meta: doctate_common::RecordingMeta = serde_json::from_slice(&bytes).unwrap(); + match meta.transcript { + doctate_common::Transcript::Content { text } => { + assert_eq!(text, "Patient mit Blutung im Cerebrum."); + } + other => panic!("expected Transcript::Content, got {other:?}"), + } } // -------------------- Ollama client -------------------- diff --git a/server/tests/transient_failure_retries_test.rs b/server/tests/transient_failure_retries_test.rs index 8963e6c..f4e2746 100644 --- a/server/tests/transient_failure_retries_test.rs +++ b/server/tests/transient_failure_retries_test.rs @@ -136,10 +136,8 @@ async fn transient_whisper_failure_is_reenqueued_and_recovers() { ".m4a.failed must not exist after transient error" ); assert!( - !case_dir - .join("2026-04-15T20-30-00Z.transcript.txt") - .exists(), - "no transcript yet — the 503 produced nothing" + !case_dir.join("2026-04-15T20-30-00Z.json").exists(), + "no metadata sidecar yet — the 503 produced nothing" ); // Act 2: heal re-enqueues the pending .m4a (second mock → 200). @@ -154,22 +152,26 @@ async fn transient_whisper_failure_is_reenqueued_and_recovers() { ) .await; - // Assert: the transcript lands. Poll because the worker runs - // asynchronously after the heal hands off the job. - let transcript = case_dir.join("2026-04-15T20-30-00Z.transcript.txt"); + // Assert: the recording metadata sidecar lands. Poll because the + // worker runs asynchronously after the heal hands off the job. + let meta_path = case_dir.join("2026-04-15T20-30-00Z.json"); let start = std::time::Instant::now(); - while !transcript.exists() { + while !meta_path.exists() { if start.elapsed() > Duration::from_secs(10) { panic!( - "transcript did not appear within 10s (whisper hits: {})", + "meta sidecar did not appear within 10s (whisper hits: {})", whisper.received_requests().await.unwrap().len() ); } tokio::time::sleep(Duration::from_millis(25)).await; } - let body = std::fs::read_to_string(&transcript).unwrap(); - assert_eq!(body, "recovered text"); + let bytes = std::fs::read(&meta_path).unwrap(); + let meta: doctate_common::RecordingMeta = serde_json::from_slice(&bytes).unwrap(); + match meta.transcript { + doctate_common::Transcript::Content { text } => assert_eq!(text, "recovered text"), + other => panic!("expected Transcript::Content, got {other:?}"), + } // Teardown: close the worker channel so the background task exits. drop(tx_t);