Brummel/doctate
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor recording metadata to JSON sidecar

Browse Source 
Replaces the `.transcript.txt` sidecar with a structured `.json` file
for recording metadata. This change consolidates transcript text,
duration, and other potential metadata into a single, extensible JSON
object.

This also refactors the `TranscriptState` enum to better represent the
on-disk state (absence of file means pending) and the in-memory
representation. The `Transcript` enum now specifically models the
terminal outcomes of the transcriber (`Silent` or `Content`).

The commit includes updates to documentation, data structures, path
handling, and various tests to align with the new metadata format.
This commit is contained in:
Brummel
2026-04-27 12:48:25 +02:00
parent a510c20e75
commit 66b3b7e4c8
20 changed files with 637 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/tests/analyze_test.rs
+5 -5
View File
@@ -1205,7 +1205,7 @@ async fn reset_case_clears_derived_and_unfails_audio() {
assert!(dir.join("2026-04-15T09-05-00Z.m4a").exists());
assert!(!dir.join("2026-04-15T09-05-00Z.m4a.failed").exists());
// Derived artefacts gone.
assert!(!dir.join("2026-04-15T09-00-00Z.transcript.txt").exists());
assert!(!dir.join("2026-04-15T09-00-00Z.json").exists());
assert!(!dir.join(ONELINER_FILENAME).exists());
assert!(!dir.join(DOCUMENT_FILE).exists());
assert!(!dir.join(ANALYSIS_INPUT_FILE).exists());
@@ -1235,7 +1235,7 @@ async fn reset_case_requires_admin() {
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
// Nothing touched.
assert!(dir.join("2026-04-15T09-00-00Z.transcript.txt").exists());
assert!(dir.join("2026-04-15T09-00-00Z.json").exists());
assert!(dir.join(DOCUMENT_FILE).exists());
}
@@ -1282,8 +1282,8 @@ async fn bulk_reset_processes_multiple_cases_admin_only() {
assert_eq!(resp.status(), StatusCode::SEE_OTHER);
assert!(!dir_a.join(DOCUMENT_FILE).exists());
assert!(!dir_b.join(DOCUMENT_FILE).exists());
assert!(!dir_a.join("2026-04-15T10-00-00Z.transcript.txt").exists());
assert!(!dir_b.join("2026-04-15T10-05-00Z.transcript.txt").exists());
assert!(!dir_a.join("2026-04-15T10-00-00Z.json").exists());
assert!(!dir_b.join("2026-04-15T10-05-00Z.json").exists());
// Non-admin: bulk reset is rejected, dr_b's case untouched.
let (cookie_doctor, csrf_doctor) = login_with_csrf(&app, &store, "dr_b", "s").await;
@@ -1299,7 +1299,7 @@ async fn bulk_reset_processes_multiple_cases_admin_only() {
.unwrap();
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
assert!(dir_c.join(DOCUMENT_FILE).exists());
assert!(dir_c.join("2026-04-15T11-00-00Z.transcript.txt").exists());
assert!(dir_c.join("2026-04-15T11-00-00Z.json").exists());
}
/// Regression guard: the bulk route handles *all* actions (including
server/tests/common/mod.rs
+3 -1
View File
@@ -27,13 +27,15 @@ pub mod users;
// Re-exports for ergonomic `use common::*;` in most test files.
pub use artefacts::{ANALYSIS_INPUT_FILE, CLOSE_MARKER, DOCUMENT_FILE, ONELINER_FILENAME};
pub use config::{TestConfig, unique_tmpdir};
pub use doctate_common::Transcript;
pub use http::{
body_bytes, body_json, body_string, csrf_form_post, form_post, get_with_api_key,
get_with_api_key_if_none_match, get_with_cookie, header_opt, header_str, multipart_upload_body,
};
pub use seed::{
past_rfc3339, seed_case, seed_oneliner_ready, seed_oneliner_state, seed_recording,
seed_recording_with_age, seed_recording_with_sidecars, write_closed_marker,
seed_recording_meta, seed_recording_with_age, seed_recording_with_sidecars,
write_closed_marker,
};
pub use session::{extract_session_cookie, login, login_request, login_with_csrf};
pub use users::{
server/tests/common/seed.rs
+54 -11
View File
@@ -11,8 +11,8 @@
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime};
use doctate_common::ONELINER_FILENAME;
use doctate_common::oneliners::OnelinerState;
use doctate_common::{ONELINER_FILENAME, RECORDING_META_SUFFIX, RecordingMeta, Transcript};
use doctate_server::paths::CLOSE_MARKER;
use filetime::FileTime;
@@ -26,28 +26,71 @@ pub fn seed_case(data_path: &Path, slug: &str, case_id: &str) -> PathBuf {
}
/// Write `<stem>.m4a` with placeholder bytes, plus optionally a
/// `<stem>.transcript.txt`. `stem` must be the full RFC3339-like
/// filename stem (e.g. `2026-04-15T10-00-00Z`); pass the same prefix
/// you want to read back later. Returns the audio filename (no path)
/// so the caller can feed it back to the delete endpoint.
/// `<stem>.json` sidecar that mirrors the transcribe outcome.
///
/// `transcript`:
/// - `None` → no sidecar written (recording stays in `Pending` state).
/// - `Some(text)` with whitespace-only `text` → sidecar with
/// `Transcript::Silent` (matches the old `transcript.txt = ""` rule).
/// - `Some(text)` non-empty → sidecar with `Transcript::Content`.
///
/// Returns the audio filename (no path) so the caller can feed it back
/// to the delete endpoint.
pub fn seed_recording(case_dir: &Path, stem: &str, transcript: Option<&str>) -> String {
let filename = format!("{stem}.m4a");
std::fs::write(case_dir.join(&filename), b"audio-bytes").unwrap();
if let Some(text) = transcript {
std::fs::write(case_dir.join(format!("{stem}.transcript.txt")), text).unwrap();
seed_recording_meta(case_dir, stem, transcript_for(text), None);
}
filename
}
/// Like [`seed_recording`] but additionally writes the `.duration.txt`
/// sidecar — used by the delete-recording tests that assert every
/// sidecar is cleaned up.
/// Like [`seed_recording`] but the sidecar carries a real duration
/// (`duration_seconds: Some(42)`). Used by tests that assert the
/// duration is rendered or cleaned up.
pub fn seed_recording_with_sidecars(case_dir: &Path, stem: &str, transcript: &str) -> String {
let filename = seed_recording(case_dir, stem, Some(transcript));
std::fs::write(case_dir.join(format!("{stem}.duration.txt")), "42").unwrap();
let filename = format!("{stem}.m4a");
std::fs::write(case_dir.join(&filename), b"audio-bytes").unwrap();
seed_recording_meta(case_dir, stem, transcript_for(transcript), Some(42));
filename
}
/// Write a fully-specified `<stem>.json` next to an already-seeded
/// `<stem>.m4a`. Use when a test needs precise control over the
/// transcript variant or duration; the higher-level helpers above
/// cover the common cases.
pub fn seed_recording_meta(
case_dir: &Path,
stem: &str,
transcript: Transcript,
duration_seconds: Option<u32>,
) {
let meta = RecordingMeta {
transcript,
duration_seconds,
};
let bytes = serde_json::to_vec(&meta).unwrap();
std::fs::write(
case_dir.join(format!("{stem}.{RECORDING_META_SUFFIX}")),
bytes,
)
.unwrap();
}
/// Map a free-text transcript to the `Transcript` variant the worker
/// would have produced for it: whitespace-only → `Silent`, otherwise
/// `Content`. Mirrors the old `transcript.txt = "" ⇒ Silent` rule so
/// existing tests keep their meaning.
fn transcript_for(text: &str) -> Transcript {
if text.trim().is_empty() {
Transcript::Silent
} else {
Transcript::Content {
text: text.to_owned(),
}
}
}
/// Write an m4a with an artificial `mtime` in the past (relative to
/// now). Used by the retention-sweep tests to simulate "recorded N days
/// ago" without a clock abstraction.
server/tests/delete_recording_test.rs
+6 -14
View File
@@ -1,7 +1,7 @@
//! Per-recording delete endpoint: POST /web/cases/{case_id}/recordings/delete.
//!
//! Verifies the endpoint:
//! - removes the audio file and its transcript / duration sidecars
//! - removes the audio file and its `<stem>.json` metadata sidecar
//! - invalidates derived artefacts (oneliner.json, document.md, analysis_input.json)
//! - leaves other recordings in the same case untouched
//! - rejects path-traversal filenames with 400
@@ -68,26 +68,18 @@ async fn delete_recording_removes_file_and_sidecars_and_invalidates_derived() {
"delete should redirect back to the recording list, not the case list"
);
// Victim + its sidecars are gone.
// Victim + its sidecar are gone.
assert!(!case_dir.join(&victim).exists(), "m4a should be deleted");
assert!(
!case_dir
.join("2026-04-19T10-00-00Z.transcript.txt")
.exists(),
"transcript sidecar should be deleted"
);
assert!(
!case_dir.join("2026-04-19T10-00-00Z.duration.txt").exists(),
"duration sidecar should be deleted"
!case_dir.join("2026-04-19T10-00-00Z.json").exists(),
"metadata sidecar should be deleted"
);
// Survivor is intact.
assert!(case_dir.join(&survivor).exists(), "other m4a must remain");
assert!(
case_dir
.join("2026-04-19T11-00-00Z.transcript.txt")
.exists(),
"other transcript must remain"
case_dir.join("2026-04-19T11-00-00Z.json").exists(),
"other metadata sidecar must remain"
);
// Derived artefacts are invalidated.
server/tests/oneliner_heal_decoupled_test.rs
+12 -9
View File
@@ -26,17 +26,20 @@ use wiremock::{Mock, MockServer, ResponseTemplate};
use common::{ONELINER_FILENAME, TestConfig};
fn write_transcript(case_dir: &Path) {
// Seed an m4a + transcript pair, matching the worker's on-disk
// layout. The oneliner worker iterates `.m4a` files and resolves
// the sidecar from the stem; a stray transcript without an m4a
// would be invisible to it.
// Seed an m4a + metadata sidecar pair, matching the worker's
// on-disk layout. The oneliner worker iterates `.m4a` files and
// resolves the sidecar from the stem; a stray sidecar without an
// m4a would be invisible to it.
let stem = "2026-04-16T10-00-00Z";
std::fs::write(case_dir.join(format!("{stem}.m4a")), b"audio").unwrap();
std::fs::write(
case_dir.join(format!("{stem}.transcript.txt")),
"Brustschmerz links, seit heute morgen.",
)
.unwrap();
common::seed_recording_meta(
case_dir,
stem,
common::Transcript::Content {
text: "Brustschmerz links, seit heute morgen.".to_owned(),
},
None,
);
}
#[tokio::test]
server/tests/oneliner_override_test.rs
+16 -10
View File
@@ -250,11 +250,14 @@ async fn early_latch_skips_llm_call_when_manual_set() {
// Also seed a transcript so update_oneliner has content to pass to LLM
// (would call without latch).
std::fs::write(case_dir.join("2026-04-26T11-00-00Z.m4a"), b"audio-bytes").unwrap();
std::fs::write(
case_dir.join("2026-04-26T11-00-00Z.transcript.txt"),
"Patient hat Fieber.",
)
.unwrap();
common::seed_recording_meta(
&case_dir,
"2026-04-26T11-00-00Z",
common::Transcript::Content {
text: "Patient hat Fieber.".into(),
},
None,
);
let locks = OnelinerLocks::new();
let events_tx = doctate_server::events::channel();
@@ -316,11 +319,14 @@ async fn re_check_drops_llm_result_when_manual_appears_during_regen() {
// Seed transcript so update_oneliner enters the LLM branch.
std::fs::write(case_dir.join("2026-04-26T11-00-00Z.m4a"), b"audio-bytes").unwrap();
std::fs::write(
case_dir.join("2026-04-26T11-00-00Z.transcript.txt"),
"Patient klagt über Schwindel.",
)
.unwrap();
common::seed_recording_meta(
&case_dir,
"2026-04-26T11-00-00Z",
common::Transcript::Content {
text: "Patient klagt über Schwindel.".into(),
},
None,
);
let locks = OnelinerLocks::new();
let events_tx = doctate_server::events::channel();
server/tests/silent_case_empty_test.rs
+3 -3
View File
@@ -28,11 +28,11 @@ use wiremock::{Mock, MockServer, ResponseTemplate};
use common::{ONELINER_FILENAME, TestConfig};
/// Seed an m4a plus a silent (0-byte) transcript sidecar — the on-disk
/// shape of a recording that whisper classified as silence.
/// Seed an m4a plus a silent metadata sidecar — the on-disk shape of
/// a recording that whisper classified as silence.
fn seed_silent_recording(case_dir: &Path, stem: &str) {
std::fs::write(case_dir.join(format!("{stem}.m4a")), b"audio").unwrap();
std::fs::write(case_dir.join(format!("{stem}.transcript.txt")), b"").unwrap();
common::seed_recording_meta(case_dir, stem, common::Transcript::Silent, None);
}
#[tokio::test]
server/tests/transcribe_test.rs
+22 -9
View File
@@ -236,7 +236,14 @@ async fn recovery_enqueues_only_pending_recordings() {
std::fs::create_dir_all(&case_a).unwrap();
std::fs::write(case_a.join("2026-04-10T10-00-00Z.m4a"), b"x").unwrap();
std::fs::write(case_a.join("2026-04-11T10-00-00Z.m4a"), b"x").unwrap();
std::fs::write(case_a.join("2026-04-11T10-00-00Z.transcript.txt"), b"done").unwrap();
common::seed_recording_meta(
&case_a,
"2026-04-11T10-00-00Z",
common::Transcript::Content {
text: "done".into(),
},
None,
);
// User 2, case with one pending .m4a.
let case_b = root.join("dr_b/bbbb");
@@ -380,14 +387,14 @@ async fn worker_leaves_m4a_intact_on_transient_whisper_error() {
"transient error must not mark {} as failed",
failed.display()
);
// And no transcript yet either (the 503 produced no text).
let transcript = case_dir.join("2026-04-13T10-31-00Z.transcript.txt");
assert!(!transcript.exists(), "no transcript expected on 503");
// And no metadata sidecar yet either (the 503 produced no text).
let meta = case_dir.join("2026-04-13T10-31-00Z.json");
assert!(!meta.exists(), "no recording meta expected on 503");
}
/// The worker must route the Whisper response through the Gazetteer
/// before persisting. Mock returns the drift form "Zerebrum"; the
/// persisted `.transcript.txt` must contain the canonical "Cerebrum".
/// persisted `<stem>.json` must carry the canonical "Cerebrum".
#[tokio::test]
async fn transcribe_worker_normalizes_whisper_output() {
let server = MockServer::start().await;
@@ -438,10 +445,16 @@ async fn transcribe_worker_normalizes_whisper_output() {
)
.await;
let transcript_path = case_dir.join("2026-04-13T10-30-00Z.transcript.txt");
assert!(transcript_path.exists(), "transcript missing");
let transcript = std::fs::read_to_string(&transcript_path).unwrap();
assert_eq!(transcript, "Patient mit Blutung im Cerebrum.");
let meta_path = case_dir.join("2026-04-13T10-30-00Z.json");
assert!(meta_path.exists(), "recording meta missing");
let bytes = std::fs::read(&meta_path).unwrap();
let meta: doctate_common::RecordingMeta = serde_json::from_slice(&bytes).unwrap();
match meta.transcript {
doctate_common::Transcript::Content { text } => {
assert_eq!(text, "Patient mit Blutung im Cerebrum.");
}
other => panic!("expected Transcript::Content, got {other:?}"),
}
}
// -------------------- Ollama client --------------------
server/tests/transient_failure_retries_test.rs
+13 -11
View File
@@ -136,10 +136,8 @@ async fn transient_whisper_failure_is_reenqueued_and_recovers() {
".m4a.failed must not exist after transient error"
);
assert!(
!case_dir
.join("2026-04-15T20-30-00Z.transcript.txt")
.exists(),
"no transcript yet — the 503 produced nothing"
!case_dir.join("2026-04-15T20-30-00Z.json").exists(),
"no metadata sidecar yet — the 503 produced nothing"
);
// Act 2: heal re-enqueues the pending .m4a (second mock → 200).
@@ -154,22 +152,26 @@ async fn transient_whisper_failure_is_reenqueued_and_recovers() {
)
.await;
// Assert: the transcript lands. Poll because the worker runs
// asynchronously after the heal hands off the job.
let transcript = case_dir.join("2026-04-15T20-30-00Z.transcript.txt");
// Assert: the recording metadata sidecar lands. Poll because the
// worker runs asynchronously after the heal hands off the job.
let meta_path = case_dir.join("2026-04-15T20-30-00Z.json");
let start = std::time::Instant::now();
while !transcript.exists() {
while !meta_path.exists() {
if start.elapsed() > Duration::from_secs(10) {
panic!(
"transcript did not appear within 10s (whisper hits: {})",
"meta sidecar did not appear within 10s (whisper hits: {})",
whisper.received_requests().await.unwrap().len()
);
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
let body = std::fs::read_to_string(&transcript).unwrap();
assert_eq!(body, "recovered text");
let bytes = std::fs::read(&meta_path).unwrap();
let meta: doctate_common::RecordingMeta = serde_json::from_slice(&bytes).unwrap();
match meta.transcript {
doctate_common::Transcript::Content { text } => assert_eq!(text, "recovered text"),
other => panic!("expected Transcript::Content, got {other:?}"),
}
// Teardown: close the worker channel so the background task exits.
drop(tx_t);