Refactor recording metadata to JSON sidecar

Replaces the `.transcript.txt` sidecar with a structured `.json` file
for recording metadata. This change consolidates transcript text,
duration, and other potential metadata into a single, extensible JSON
object.

This also refactors the `TranscriptState` enum to better represent the
on-disk state (absence of file means pending) and the in-memory
representation. The `Transcript` enum now specifically models the
terminal outcomes of the transcriber (`Silent` or `Content`).

The commit includes updates to documentation, data structures, path
handling, and various tests to align with the new metadata format.
This commit is contained in:
2026-04-27 12:48:25 +02:00
parent a510c20e75
commit 66b3b7e4c8
20 changed files with 637 additions and 335 deletions
+39 -21
View File
@@ -318,6 +318,31 @@ mod tests {
fs::write(path, b"").await.unwrap();
}
/// Seed a `<stem>.json` next to a (presumed already-touched)
/// `<stem>.m4a`. Variant is `Transcript::Silent` because these
/// tests historically used an empty `.transcript.txt`, which
/// mapped to `TranscriptState::Silent`. Tests that need real
/// content call `seed_meta_with` with the desired variant.
async fn seed_meta(case_dir: &std::path::Path, stem: &str) {
crate::paths::write_recording_meta_sync(
case_dir,
stem,
doctate_common::Transcript::Silent,
None,
);
}
async fn seed_meta_with(case_dir: &std::path::Path, stem: &str, text: &str) {
crate::paths::write_recording_meta_sync(
case_dir,
stem,
doctate_common::Transcript::Content {
text: text.to_owned(),
},
None,
);
}
fn set_mtime(path: &std::path::Path, t: SystemTime) {
set_file_mtime(path, FileTime::from_system_time(t)).unwrap();
}
@@ -335,7 +360,7 @@ mod tests {
async fn missing_transcript_skips() {
let dir = TempDir::new().unwrap();
touch(&dir.path().join("2026-01-01T10-00-00Z.m4a")).await;
touch(&dir.path().join("2026-01-01T10-00-00Z.transcript.txt")).await;
seed_meta(dir.path(), "2026-01-01T10-00-00Z").await;
touch(&dir.path().join("2026-01-01T11-00-00Z.m4a")).await;
assert_eq!(
evaluate_case(dir.path()).await,
@@ -347,7 +372,7 @@ mod tests {
async fn all_transcribed_no_document_enqueues() {
let dir = TempDir::new().unwrap();
touch(&dir.path().join("2026-01-01T10-00-00Z.m4a")).await;
touch(&dir.path().join("2026-01-01T10-00-00Z.transcript.txt")).await;
seed_meta(dir.path(), "2026-01-01T10-00-00Z").await;
assert_eq!(evaluate_case(dir.path()).await, AutoDecision::Enqueue);
}
@@ -356,7 +381,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let m4a = dir.path().join("2026-01-01T10-00-00Z.m4a");
touch(&m4a).await;
touch(&dir.path().join("2026-01-01T10-00-00Z.transcript.txt")).await;
seed_meta(dir.path(), "2026-01-01T10-00-00Z").await;
let doc = dir.path().join(DOCUMENT_FILE);
touch(&doc).await;
@@ -375,7 +400,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let m4a = dir.path().join("2026-01-01T10-00-00Z.m4a");
touch(&m4a).await;
touch(&dir.path().join("2026-01-01T10-00-00Z.transcript.txt")).await;
seed_meta(dir.path(), "2026-01-01T10-00-00Z").await;
let doc = dir.path().join(DOCUMENT_FILE);
touch(&doc).await;
@@ -390,7 +415,7 @@ mod tests {
async fn analysis_input_present_skips() {
let dir = TempDir::new().unwrap();
touch(&dir.path().join("2026-01-01T10-00-00Z.m4a")).await;
touch(&dir.path().join("2026-01-01T10-00-00Z.transcript.txt")).await;
seed_meta(dir.path(), "2026-01-01T10-00-00Z").await;
touch(&dir.path().join(ANALYSIS_INPUT_FILE)).await;
assert_eq!(
evaluate_case(dir.path()).await,
@@ -403,7 +428,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let m4a = dir.path().join("2026-01-01T10-00-00Z.m4a");
touch(&m4a).await;
touch(&dir.path().join("2026-01-01T10-00-00Z.transcript.txt")).await;
seed_meta(dir.path(), "2026-01-01T10-00-00Z").await;
let base = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000);
set_mtime(&m4a, base);
@@ -431,7 +456,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let m4a = dir.path().join("2026-01-01T10-00-00Z.m4a");
touch(&m4a).await;
touch(&dir.path().join("2026-01-01T10-00-00Z.transcript.txt")).await;
seed_meta(dir.path(), "2026-01-01T10-00-00Z").await;
let base = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000);
set_mtime(&m4a, base);
@@ -478,12 +503,12 @@ mod tests {
fs::write(eligible.join("2026-01-01T10-00-00Z.m4a"), b"fake")
.await
.unwrap();
fs::write(
eligible.join("2026-01-01T10-00-00Z.transcript.txt"),
b"patient mit brustschmerz",
seed_meta_with(
&eligible,
"2026-01-01T10-00-00Z",
"patient mit brustschmerz",
)
.await
.unwrap();
.await;
// Not eligible: has a fresh document.
let current_id = "22222222-2222-2222-2222-222222222222";
@@ -491,12 +516,7 @@ mod tests {
fs::create_dir_all(&current).await.unwrap();
let m4a = current.join("2026-01-01T10-00-00Z.m4a");
fs::write(&m4a, b"fake").await.unwrap();
fs::write(
current.join("2026-01-01T10-00-00Z.transcript.txt"),
b"stable",
)
.await
.unwrap();
seed_meta_with(&current, "2026-01-01T10-00-00Z", "stable").await;
let doc = current.join(DOCUMENT_FILE);
fs::write(&doc, b"doc").await.unwrap();
let base = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000);
@@ -509,9 +529,7 @@ mod tests {
fs::write(junk.join("2026-01-01T10-00-00Z.m4a"), b"fake")
.await
.unwrap();
fs::write(junk.join("2026-01-01T10-00-00Z.transcript.txt"), b"x")
.await
.unwrap();
seed_meta_with(&junk, "2026-01-01T10-00-00Z", "x").await;
let (tx, mut rx) = analyze_channel();
let events_tx = events_channel();
+112 -17
View File
@@ -1,13 +1,47 @@
use std::path::{Path, PathBuf};
use doctate_common::TranscriptState;
use doctate_common::oneliners::{ONELINER_FILENAME, OnelinerState};
use doctate_common::{RECORDING_META_SUFFIX, RecordingMeta, TranscriptState};
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
use tracing::info;
use crate::oneliner_locks::OnelinerLocks;
/// Atomically write `value` as JSON to `final_path`.
///
/// Writes to `<final_path>.tmp` first, then `rename`s into place.
/// `rename` within the same filesystem is atomic on every POSIX target
/// we run on, so concurrent readers either see the previous full file
/// or the new full file — never a half-written one. Caller is
/// responsible for the destination's parent directory existing.
///
/// Used by every metadata sidecar (oneliner state, per-recording meta,
/// …) so the on-disk format never has partial documents the parser
/// would choke on.
pub async fn atomic_write_json<T: Serialize>(final_path: &Path, value: &T) -> std::io::Result<()> {
let tmp_path = match final_path.file_name() {
Some(name) => {
let mut tmp_name = name.to_owned();
tmp_name.push(".tmp");
final_path.with_file_name(tmp_name)
}
// No file_name on a path means we cannot derive a sibling tmp
// file — bail out with InvalidInput rather than silently
// writing to a wrong place.
None => {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"atomic_write_json: target path has no file name",
));
}
};
let payload = serde_json::to_vec(value)
.map_err(|e| std::io::Error::other(format!("serialize json: {e}")))?;
tokio::fs::write(&tmp_path, &payload).await?;
tokio::fs::rename(&tmp_path, final_path).await
}
/// Absolute on-disk location of a case directory.
///
/// Single source of truth for the layout `<data_path>/<slug>/<case_id>/`.
@@ -86,16 +120,12 @@ pub async fn read_oneliner_state(
(Some(state), mtime)
}
/// Write `state` atomically to `case_dir/oneliner.json` via
/// `write(tmp) + rename(tmp, final)`. Rename is atomic within a single
/// filesystem, so concurrent readers never see a half-serialized JSON
/// document.
/// Write `state` atomically to `case_dir/oneliner.json`. Thin wrapper
/// over [`atomic_write_json`] that fixes the filename — kept as its own
/// function so call-sites read as "write the oneliner state" rather
/// than threading the filename constant in by hand.
pub async fn write_oneliner_state(case_dir: &Path, state: &OnelinerState) -> std::io::Result<()> {
let final_path = case_dir.join(ONELINER_FILENAME);
let tmp_path = case_dir.join(format!("{ONELINER_FILENAME}.tmp"));
let payload = serde_json::to_vec(state).expect("OnelinerState is infallibly serializable");
tokio::fs::write(&tmp_path, &payload).await?;
tokio::fs::rename(&tmp_path, &final_path).await
atomic_write_json(&case_dir.join(ONELINER_FILENAME), state).await
}
/// Delete `case_dir/oneliner.json` unless it persists a manual override.
@@ -138,21 +168,86 @@ pub async fn delete_oneliner_unless_manual(
}
}
/// Load the `.transcript.txt` sidecar for a recording as a
/// Load the `<stem>.json` metadata sidecar for a recording as a
/// [`TranscriptState`].
///
/// `audio_stem_path` is the non-`.failed` stem path of the recording
/// (e.g. `case_dir/2026-04-16T16-23-38Z.m4a`) — the sidecar sits next
/// to it with extension `.transcript.txt`.
/// to it with the `.json` extension.
///
/// This is the single reader for that sidecar. Every call-site that
/// used to do `tokio::fs::read_to_string(...).ok()` or
/// This is the single reader of that sidecar's transcript field.
/// Every call-site that used to do
/// `tokio::fs::read_to_string(...).ok()` or
/// `with_extension("transcript.txt").exists()` should go through here
/// so the three-state semantics stay consistent across the codebase.
pub async fn read_transcript_state(audio_stem_path: &Path) -> TranscriptState {
let transcript_path = audio_stem_path.with_extension("transcript.txt");
let raw = tokio::fs::read_to_string(&transcript_path).await.ok();
TranscriptState::from_raw(raw.as_deref())
TranscriptState::from_meta(read_recording_meta(audio_stem_path).await.as_ref())
}
/// Path of the per-recording metadata sidecar next to `audio_stem_path`.
///
/// `audio_stem_path` is the non-`.failed` stem path of the recording
/// (e.g. `case_dir/2026-04-16T16-23-38Z.m4a`); the sidecar lives at
/// `case_dir/2026-04-16T16-23-38Z.json`.
pub fn recording_meta_path(audio_stem_path: &Path) -> PathBuf {
audio_stem_path.with_extension(RECORDING_META_SUFFIX)
}
/// Read and parse the `<stem>.json` sidecar for a recording.
///
/// Returns `None` for missing files (the typical "transcriber hasn't
/// finished yet" case) and for malformed JSON (logged at warn — a
/// rewritten sidecar will overwrite the bad one on the next worker
/// pass). Callers translate `None` to [`TranscriptState::Pending`].
pub async fn read_recording_meta(audio_stem_path: &Path) -> Option<RecordingMeta> {
let path = recording_meta_path(audio_stem_path);
let bytes = tokio::fs::read(&path).await.ok()?;
match serde_json::from_slice(&bytes) {
Ok(meta) => Some(meta),
Err(e) => {
tracing::warn!(
path = %path.display(),
error = %e,
"malformed recording meta sidecar; treating as missing"
);
None
}
}
}
/// Atomically write `<stem>.json` for a recording. Thin wrapper around
/// [`atomic_write_json`] that fixes the filename so call-sites read as
/// "persist this recording's metadata" rather than threading the
/// suffix in by hand.
pub async fn write_recording_meta_atomic(
audio_stem_path: &Path,
meta: &RecordingMeta,
) -> std::io::Result<()> {
atomic_write_json(&recording_meta_path(audio_stem_path), meta).await
}
/// Synchronous test fixture: write a `<stem>.json` directly with the
/// given transcript and duration. Mirrors what the worker would have
/// persisted at the end of its pipeline. Async helpers exist for
/// production paths; this one keeps unit-test setups terse and
/// callable from non-async contexts.
#[cfg(test)]
pub(crate) fn write_recording_meta_sync(
case_dir: &Path,
stem: &str,
transcript: doctate_common::Transcript,
duration_seconds: Option<u32>,
) {
let meta = doctate_common::RecordingMeta {
transcript,
duration_seconds,
};
let bytes = serde_json::to_vec(&meta).expect("RecordingMeta serializes infallibly");
std::fs::write(
case_dir.join(format!("{stem}.{RECORDING_META_SUFFIX}")),
bytes,
)
.expect("write recording meta sidecar in test fixture");
}
#[cfg(test)]
+27 -12
View File
@@ -11,8 +11,8 @@ use time::format_description::well_known::Rfc3339;
use tokio::io::AsyncWriteExt;
use tracing::{info, warn};
use doctate_common::TranscriptState;
use doctate_common::timestamp::{filename_stem_to_recorded_at, now_rfc3339};
use doctate_common::{RECORDING_META_SUFFIX, TranscriptState};
use crate::analyze::{
ANALYSIS_INPUT_FILE, AnalysisInput, AnalyzeJob, AnalyzeSender, DOCUMENT_FILE, RecordingInput,
@@ -171,7 +171,7 @@ fn resolve_list_return_path(headers: &HeaderMap) -> String {
/// Build an `AnalysisInput` for the given case directory.
/// Collects all `.m4a` (excluding `.m4a.failed`), reads the matching
/// `.transcript.txt` for each, skips blank transcripts, and computes
/// `<stem>.json` for each, skips blank transcripts, and computes
/// `last_recording_mtime` as the max mtime over ALL `.m4a` (including blank
/// ones — a late blank addendum still counts as activity).
pub(crate) async fn build_analysis_input(case_dir: &Path) -> Result<AnalysisInput, AppError> {
@@ -225,7 +225,7 @@ async fn read_recordings(m4as: &[(PathBuf, SystemTime)]) -> Result<Vec<Recording
));
}
TranscriptState::Silent => continue,
TranscriptState::Content(t) => t,
TranscriptState::Content { text } => text,
};
let stem = m4a_path.file_stem().and_then(|s| s.to_str()).unwrap_or("");
@@ -322,12 +322,27 @@ pub(crate) async fn reset_case_artefacts(
}
delete_oneliner_unless_manual(case_dir, locks).await?;
let mut entries = tokio::fs::read_dir(case_dir).await?;
let meta_suffix = format!(".{RECORDING_META_SUFFIX}");
while let Some(entry) = entries.next_entry().await? {
let name = entry.file_name();
let Some(s) = name.to_str() else { continue };
if s.ends_with(".transcript.txt") {
tokio::fs::remove_file(entry.path()).await?;
} else if s.ends_with(".m4a.failed") {
// Per-recording metadata sidecar: paired with `<stem>.m4a` or
// its `.failed` twin. The pair check guards case-level JSONs
// (preserved Manual oneliner, future siblings) from being
// misidentified as recording artefacts.
if let Some(stem) = s.strip_suffix(&meta_suffix) {
let paired = tokio::fs::try_exists(case_dir.join(format!("{stem}.m4a")))
.await
.unwrap_or(false)
|| tokio::fs::try_exists(case_dir.join(format!("{stem}.m4a.failed")))
.await
.unwrap_or(false);
if paired {
tokio::fs::remove_file(entry.path()).await?;
}
continue;
}
if s.ends_with(".m4a.failed") {
let new_name = &s[..s.len() - ".failed".len()];
tokio::fs::rename(entry.path(), case_dir.join(new_name)).await?;
}
@@ -601,13 +616,13 @@ pub async fn handle_delete_recording(
.trim_end_matches(".failed")
.trim_end_matches(".m4a");
// Audio and its sidecars. NotFound on any of these is fine: the
// transcript / duration may never have been written (silence, failed
// job, or the file got deleted twice).
let targets: [PathBuf; 5] = [
// Audio and its single metadata sidecar. NotFound on any of these
// is fine: the JSON sidecar may never have been written (the
// worker hadn't reached the atomic write yet, or the file got
// deleted twice).
let targets: [PathBuf; 4] = [
case_dir.join(&form.filename),
case_dir.join(format!("{stem}.transcript.txt")),
case_dir.join(format!("{stem}.duration.txt")),
case_dir.join(format!("{stem}.{RECORDING_META_SUFFIX}")),
case_dir.join(DOCUMENT_FILE),
case_dir.join(ANALYSIS_INPUT_FILE),
];
+1 -1
View File
@@ -116,7 +116,7 @@ pub async fn handle_upload(
);
// Enqueue transcription. Failure here is non-fatal: on restart the recovery
// scan will re-enqueue any .m4a without a .transcript.txt sibling.
// scan will re-enqueue any .m4a without a <stem>.json sibling.
match transcribe_tx.try_send(TranscribeJob {
audio_path: file_path.clone(),
user_slug: user.slug.clone(),
+41 -44
View File
@@ -10,7 +10,6 @@ use doctate_common::TranscriptState;
use doctate_common::timestamp::filename_stem_to_recorded_at;
use tokio::io::{AsyncReadExt, AsyncSeekExt};
use tokio::task::JoinSet;
use tracing::debug;
use crate::config::Config;
use crate::error::AppError;
@@ -22,9 +21,11 @@ pub(crate) struct RecordingView {
/// RFC3339 UTC timestamp derived from the filename stem. Empty only
/// on malformed filenames, which should not occur in production.
pub(crate) recorded_at_iso: String,
/// Container duration in whole seconds, read from the `<stem>.duration.txt`
/// sidecar. `None` means neither the worker nor the lazy-backfill path
/// produced one — the UI then falls back to browser `preload="metadata"`.
/// Container duration in whole seconds, read from the `<stem>.json`
/// sidecar's `duration_seconds` field. `None` means the worker
/// hasn't finished yet (no sidecar) and the in-flight lazy ffprobe
/// could not produce one either — the UI then falls back to
/// browser `preload="metadata"`.
pub(crate) duration_seconds: Option<u32>,
/// Three-way transcript state — `Pending` (not yet transcribed),
/// `Silent` (transcriber classified as silence), or `Content(text)`.
@@ -164,16 +165,15 @@ fn stem_of(filename: &str) -> &str {
.trim_end_matches(".m4a")
}
/// One entry in progress between the directory enumeration and the optional
/// parallel ffprobe backfill. Holds the resolved paths so the spawned task
/// doesn't have to know about the case_dir.
/// One entry in progress between the directory enumeration and the
/// in-flight ffprobe backfill. The audio path is kept around so the
/// spawned task can probe directly without re-deriving the location.
struct RawRecording {
filename: String,
failed: bool,
transcript: TranscriptState,
duration_seconds: Option<u32>,
audio_path: PathBuf,
sidecar_path: PathBuf,
}
pub(crate) async fn scan_recordings(case_dir: &Path) -> Vec<RecordingView> {
@@ -193,17 +193,14 @@ pub(crate) async fn scan_recordings(case_dir: &Path) -> Vec<RecordingView> {
if !is_audio {
continue;
}
// Transcript and duration sidecar both hang off the non-failed stem path;
// a `.failed` file never produced a transcript but the lookup still works.
// Transcript and duration both come from the per-recording
// metadata sidecar; a `.failed` file never produced one, but
// the lookup still works (returns None → Pending state).
let stem_file = filename.trim_end_matches(".failed");
let stem_path = case_dir.join(stem_file);
let transcript = paths::read_transcript_state(&stem_path).await;
let sidecar_path = stem_path.with_extension("duration.txt");
let duration_seconds = tokio::fs::read_to_string(&sidecar_path)
.await
.ok()
.and_then(|s| s.trim().parse::<u32>().ok());
let meta = paths::read_recording_meta(&stem_path).await;
let duration_seconds = meta.as_ref().and_then(|m| m.duration_seconds);
let transcript = TranscriptState::from_meta(meta.as_ref());
let audio_path = case_dir.join(&filename);
raws.push(RawRecording {
@@ -212,25 +209,25 @@ pub(crate) async fn scan_recordings(case_dir: &Path) -> Vec<RecordingView> {
transcript,
duration_seconds,
audio_path,
sidecar_path,
});
}
// Lazy backfill: spawn ffprobe in parallel for recordings without a sidecar.
// `.failed` entries are skipped — ffprobe on a corrupt file is unlikely to
// succeed and the UI doesn't need a duration there.
// Read-only lazy backfill: while the transcribe worker is still in
// flight (no sidecar yet), run ffprobe in parallel for the UI render
// so the recordings list shows a duration during the transcription
// window. The result is *not* persisted — the worker is the single
// source of truth for `<stem>.json` and writes both transcript and
// duration in one atomic step at the end of its pipeline.
// `.failed` entries are skipped: ffprobe on a corrupt file is
// unlikely to succeed and the UI doesn't need a duration there.
let mut js: JoinSet<Option<(usize, u32)>> = JoinSet::new();
for (idx, r) in raws.iter().enumerate() {
if r.duration_seconds.is_some() || r.failed {
continue;
}
let audio_path = r.audio_path.clone();
let sidecar_path = r.sidecar_path.clone();
js.spawn(async move {
let secs = ffmpeg::probe_duration_seconds(&audio_path).await.ok()?;
if let Err(e) = tokio::fs::write(&sidecar_path, secs.to_string()).await {
debug!(sidecar = %sidecar_path.display(), error = %e, "duration sidecar write failed");
}
Some((idx, secs))
});
}
@@ -266,20 +263,19 @@ mod tests {
use tempfile::tempdir;
#[tokio::test]
async fn scan_reads_duration_sidecar_and_derives_iso() {
async fn scan_reads_meta_sidecar_and_derives_iso() {
let dir = tempdir().unwrap();
tokio::fs::write(dir.path().join("2026-04-19T10-00-00Z.m4a"), b"audio")
.await
.unwrap();
tokio::fs::write(dir.path().join("2026-04-19T10-00-00Z.duration.txt"), "42")
.await
.unwrap();
tokio::fs::write(
dir.path().join("2026-04-19T10-00-00Z.transcript.txt"),
"hallo",
)
.await
.unwrap();
crate::paths::write_recording_meta_sync(
dir.path(),
"2026-04-19T10-00-00Z",
doctate_common::Transcript::Content {
text: "hallo".into(),
},
Some(42),
);
let got = scan_recordings(dir.path()).await;
assert_eq!(got.len(), 1);
@@ -335,21 +331,22 @@ mod tests {
}
#[tokio::test]
async fn scan_skips_corrupt_sidecar() {
async fn scan_handles_malformed_meta_sidecar() {
let dir = tempdir().unwrap();
tokio::fs::write(dir.path().join("2026-04-19T12-00-00Z.m4a"), b"audio")
.await
.unwrap();
tokio::fs::write(
dir.path().join("2026-04-19T12-00-00Z.duration.txt"),
"not-a-number",
)
.await
.unwrap();
// No ffprobe available in unit-test context (or it fails on "audio"), so
// duration_seconds remains None — exactly what we want to assert.
// Garbage JSON: read_recording_meta logs a warning and returns
// None, so the recording surfaces as Pending with no duration.
// The lazy ffprobe backfill would normally produce a duration,
// but ffprobe on the placeholder bytes fails — leaving None,
// exactly what we want to assert.
tokio::fs::write(dir.path().join("2026-04-19T12-00-00Z.json"), "not-a-json")
.await
.unwrap();
let got = scan_recordings(dir.path()).await;
assert_eq!(got.len(), 1);
assert!(got[0].duration_seconds.is_none());
assert!(got[0].transcript.is_pending());
}
}
+61 -38
View File
@@ -1,5 +1,6 @@
use std::path::{Path, PathBuf};
use doctate_common::RECORDING_META_SUFFIX;
use doctate_common::oneliners::OnelinerState;
use tracing::{info, warn};
@@ -35,8 +36,9 @@ pub async fn scan_and_enqueue(data_path: &Path, tx: &TranscribeSender) {
}
/// Scan a single `<user_root>/<case>/*.m4a` set and enqueue every recording
/// whose `.transcript.txt` is missing. Returns the number sent. Channel send
/// is awaited (bounded channel); only fails if the worker is gone.
/// whose metadata sidecar (`<stem>.json`) is missing. Returns the number
/// sent. Channel send is awaited (bounded channel); only fails if the
/// worker is gone.
pub async fn enqueue_pending_for_user(
user_root: &Path,
slug: &str,
@@ -165,13 +167,17 @@ pub(crate) async fn cases_needing_oneliner_in(user_root: &Path) -> Vec<PathBuf>
/// meaning the transcribe pipeline will produce no further artefacts
/// and `update_oneliner` has enough information to settle the case.
///
/// Three terminal shapes count:
/// - `.transcript.txt` with content → [`TranscriptState::Content`]
/// - `.transcript.txt` empty → [`TranscriptState::Silent`]
/// - `.m4a.failed` → permanent transcription failure (ffmpeg remux or
/// Whisper 4xx). Transient Whisper errors leave the file as plain
/// `.m4a` without sidecar (Pending), so they do *not* match here —
/// the page-load heal re-enqueues them on the next refresh.
/// Two terminal shapes count:
/// - `<stem>.json` next to `<stem>.m4a` → transcriber wrote its
/// metadata sidecar (carries either Silent or Content). The
/// `<stem>.m4a` pairing is required because the case directory also
/// holds case-level JSONs (`oneliner.json`, `analysis_input.json`)
/// that must not be misread as a recording sidecar.
/// - `<stem>.m4a.failed` → permanent transcription failure (ffmpeg
/// remux or Whisper 4xx). Transient Whisper errors leave the file
/// as plain `.m4a` without sidecar (Pending), so they do *not*
/// match here — the page-load heal re-enqueues them on the next
/// refresh.
///
/// Cases where every recording is still `Pending` (plain `.m4a`, no
/// sidecar, not marked failed) return false: the transcriber hasn't
@@ -186,9 +192,19 @@ async fn has_any_transcript(case_dir: &Path) -> bool {
return false;
};
while let Ok(Some(f)) = files.next_entry().await {
if f.file_name()
.to_str()
.is_some_and(|s| s.ends_with(".transcript.txt") || s.ends_with(".m4a.failed"))
let Some(name) = f.file_name().to_str().map(str::to_owned) else {
continue;
};
if name.ends_with(".m4a.failed") {
return true;
}
// A per-recording meta sidecar's stem matches its audio's
// stem. Pair the JSON with `<stem>.m4a` to filter out
// case-level JSONs (oneliner.json, analysis_input.json).
if let Some(stem) = name.strip_suffix(&format!(".{RECORDING_META_SUFFIX}"))
&& tokio::fs::try_exists(case_dir.join(format!("{stem}.m4a")))
.await
.unwrap_or(false)
{
return true;
}
@@ -246,16 +262,33 @@ pub async fn regenerate_missing_oneliners_for_user(
#[cfg(test)]
mod tests {
use super::*;
use doctate_common::Transcript;
use tempfile::tempdir;
/// Seed `<stem>.m4a` plus its `<stem>.json` metadata sidecar — the
/// shape `has_any_transcript` recognises as terminal. The pair
/// (audio + sidecar) is required because recovery filters
/// case-level JSONs from the per-recording set by checking for an
/// audio sibling.
async fn seed_recording(case: &Path, stem: &str, transcript: Transcript) {
tokio::fs::write(case.join(format!("{stem}.m4a")), b"audio")
.await
.unwrap();
crate::paths::write_recording_meta_sync(case, stem, transcript, None);
}
fn content(text: &str) -> Transcript {
Transcript::Content {
text: text.to_owned(),
}
}
#[tokio::test]
async fn cases_needing_oneliner_finds_crash_survivor() {
let data = tempdir().unwrap();
let case = data.path().join("user").join("case1");
tokio::fs::create_dir_all(&case).await.unwrap();
tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), "content")
.await
.unwrap();
seed_recording(&case, "2026-04-16T10-00-00Z", content("content")).await;
let got = cases_needing_oneliner(data.path()).await;
assert_eq!(got, vec![(case, "user".to_owned())]);
@@ -273,9 +306,7 @@ mod tests {
let data = tempdir().unwrap();
let case = data.path().join("user").join("case1");
tokio::fs::create_dir_all(&case).await.unwrap();
tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), "content")
.await
.unwrap();
seed_recording(&case, "2026-04-16T10-00-00Z", content("content")).await;
write_state(
&case,
&OnelinerState::Ready {
@@ -293,9 +324,7 @@ mod tests {
let data = tempdir().unwrap();
let case = data.path().join("user").join("case1");
tokio::fs::create_dir_all(&case).await.unwrap();
tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), "content")
.await
.unwrap();
seed_recording(&case, "2026-04-16T10-00-00Z", content("content")).await;
write_state(
&case,
&OnelinerState::Empty {
@@ -312,9 +341,7 @@ mod tests {
let data = tempdir().unwrap();
let case = data.path().join("user").join("case1");
tokio::fs::create_dir_all(&case).await.unwrap();
tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), "content")
.await
.unwrap();
seed_recording(&case, "2026-04-16T10-00-00Z", content("content")).await;
write_state(
&case,
&OnelinerState::Error {
@@ -327,27 +354,25 @@ mod tests {
assert_eq!(got, vec![(case, "user".to_owned())]);
}
/// Silent-only cases (transcript file exists but is empty/whitespace)
/// are now picked up by recovery so `update_oneliner` can settle
/// them to [`OnelinerState::Empty`]. Previously they were skipped,
/// which left the UI stuck on "Generating" forever (the bug this
/// enum refactor fixes).
/// Silent-only cases (sidecar carries `Transcript::Silent`) are
/// picked up by recovery so `update_oneliner` can settle them to
/// [`OnelinerState::Empty`]. Previously they were skipped, which
/// left the UI stuck on "Generating" forever (the bug the
/// three-state model fixes).
#[tokio::test]
async fn cases_needing_oneliner_picks_up_silent_only_cases_for_empty_settlement() {
let data = tempdir().unwrap();
let case = data.path().join("user").join("case1");
tokio::fs::create_dir_all(&case).await.unwrap();
tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), " ")
.await
.unwrap();
seed_recording(&case, "2026-04-16T10-00-00Z", Transcript::Silent).await;
let got = cases_needing_oneliner(data.path()).await;
assert_eq!(got, vec![(case, "user".to_owned())]);
}
/// Fully-pending cases (no transcript file yet) are skipped —
/// Fully-pending cases (no sidecar yet) are skipped —
/// `update_oneliner` has nothing to act on until the transcriber
/// writes a sidecar.
/// writes one.
#[tokio::test]
async fn cases_needing_oneliner_skips_fully_pending_cases() {
let data = tempdir().unwrap();
@@ -356,7 +381,7 @@ mod tests {
tokio::fs::write(case.join("2026-04-16T10-00-00Z.m4a"), b"audio")
.await
.unwrap();
// No .transcript.txt — the recording is still Pending.
// No <stem>.json — the recording is still Pending.
assert!(cases_needing_oneliner(data.path()).await.is_empty());
}
@@ -366,9 +391,7 @@ mod tests {
let data = tempdir().unwrap();
let case = data.path().join("user").join("case1");
tokio::fs::create_dir_all(&case).await.unwrap();
tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), "content")
.await
.unwrap();
seed_recording(&case, "2026-04-16T10-00-00Z", content("content")).await;
// Empty JSON object is treated as a valid close marker
// (see paths::is_closed — existence of the file is enough).
tokio::fs::write(case.join(".closed"), "{}").await.unwrap();
+63 -62
View File
@@ -2,8 +2,8 @@ use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use doctate_common::TranscriptState;
use doctate_common::oneliners::OnelinerState;
use doctate_common::{RecordingMeta, Transcript, TranscriptState};
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;
use tracing::{error, info, warn};
@@ -38,9 +38,13 @@ pub async fn run(
while let Some(job) = rx.recv().await {
let _guard = BusyGuard::new(worker_busy.clone());
let audio_path = job.audio_path;
let transcript_path = audio_path.with_extension("transcript.txt");
let meta_path = paths::recording_meta_path(&audio_path);
if transcript_path.exists() {
// Idempotency gate: presence of `<stem>.json` means the
// transcriber already finished this recording (single atomic
// write at the end of the pipeline). Skip the job, freeing
// Whisper for the next one.
if meta_path.exists() {
continue;
}
@@ -65,11 +69,24 @@ pub async fn run(
}
};
// Persist the container duration as a sidecar so the UI can render it
// without the browser having to HEAD every audio file. Non-fatal — the
// recordings scan has a lazy-backfill path, and transcription is the
// critical workload here.
write_duration_sidecar(&audio_path, remuxed.path()).await;
// Probe the container duration on the remuxed copy (its `moov`
// atom sits at the start, so ffprobe returns without seeking
// to EOF). We carry the value in memory and persist it
// together with the transcript at the end of the pipeline.
// Non-fatal: a probe failure leaves `duration_seconds = None`
// and the recordings scan falls back to a read-only ffprobe
// for the UI render.
let duration_seconds = match ffmpeg::probe_duration_seconds(remuxed.path()).await {
Ok(s) => Some(s),
Err(e) => {
warn!(
audio = %audio_path.display(),
error = %e,
"ffprobe duration failed — meta will record null duration"
);
None
}
};
let text = match whisper::transcribe(
&client,
@@ -106,15 +123,31 @@ pub async fn run(
// the canonical path.
let text = vocab.replace(&text);
if let Err(e) = tokio::fs::write(&transcript_path, &text).await {
error!(transcript = %transcript_path.display(), error = %e, "writing transcript failed");
// Single atomic write: assemble the recording metadata
// (transcript outcome + duration) in memory, then materialise
// it on disk in one rename. Either readers see the previous
// state (no sidecar = pending) or the full new sidecar — never
// a half-written one.
let bytes_logged = text.len();
let transcript = if text.trim().is_empty() {
Transcript::Silent
} else {
Transcript::Content { text }
};
let meta = RecordingMeta {
transcript,
duration_seconds,
};
if let Err(e) = paths::write_recording_meta_atomic(&audio_path, &meta).await {
error!(meta = %meta_path.display(), error = %e, "writing recording meta failed");
continue;
}
info!(
audio = %audio_path.display(),
bytes = text.len(),
"Transcript written"
bytes = bytes_logged,
duration = ?duration_seconds,
"Recording meta written"
);
// Notify the live-update bus so subscribed browsers can refresh the
@@ -185,23 +218,6 @@ async fn mark_failed(audio_path: &Path, events_tx: &EventSender, user_slug: &str
}
}
/// Probe the (remuxed) audio and persist the duration next to the original file
/// as `<ts>.duration.txt`. Probes on the remuxed copy because its `moov` atom
/// sits at the start — ffprobe returns without seeking to EOF.
async fn write_duration_sidecar(audio_path: &Path, probe_path: &Path) {
let secs = match ffmpeg::probe_duration_seconds(probe_path).await {
Ok(s) => s,
Err(e) => {
warn!(audio = %audio_path.display(), error = %e, "ffprobe duration failed — UI will lazy-backfill");
return;
}
};
let sidecar = audio_path.with_extension("duration.txt");
if let Err(e) = tokio::fs::write(&sidecar, secs.to_string()).await {
warn!(sidecar = %sidecar.display(), error = %e, "writing duration sidecar failed");
}
}
/// Regenerate `case_dir/oneliner.json` from **all** `Content` transcripts
/// in the case, joined chronologically. Called after every successful
/// transcript write so later recordings can correct earlier ones (mirror
@@ -408,7 +424,7 @@ pub(crate) async fn all_transcripts_joined(case_dir: &Path) -> Option<String> {
let mut parts: Vec<String> = Vec::with_capacity(m4as.len());
for path in m4as {
if let TranscriptState::Content(text) = paths::read_transcript_state(&path).await {
if let TranscriptState::Content { text } = paths::read_transcript_state(&path).await {
let trimmed = text.trim();
if !trimmed.is_empty() {
parts.push(trimmed.to_string());
@@ -426,18 +442,27 @@ mod tests {
use super::*;
use tempfile::tempdir;
/// Helper: seed an m4a plus an optional transcript sidecar, matching
/// Helper: seed an m4a plus an optional metadata sidecar, matching
/// how the worker lays files out on disk. `all_transcripts_joined`
/// iterates `.m4a` files and reads the sidecar through the shared
/// reader, so bare transcripts without m4as would be ignored.
/// reader, so bare meta files without m4as would be ignored.
///
/// `transcript`: `None` → no sidecar (Pending); `Some(s)` with
/// whitespace-only `s` → `Transcript::Silent`; otherwise
/// `Transcript::Content`.
async fn seed(dir: &Path, stem: &str, transcript: Option<&str>) {
tokio::fs::write(dir.join(format!("{stem}.m4a")), b"audio")
.await
.unwrap();
if let Some(text) = transcript {
tokio::fs::write(dir.join(format!("{stem}.transcript.txt")), text)
.await
.unwrap();
let variant = if text.trim().is_empty() {
Transcript::Silent
} else {
Transcript::Content {
text: text.to_owned(),
}
};
paths::write_recording_meta_sync(dir, stem, variant, None);
}
}
@@ -503,15 +528,7 @@ mod tests {
#[tokio::test]
async fn has_pending_recordings_true_when_m4a_without_transcript() {
let dir = tempdir().unwrap();
tokio::fs::write(dir.path().join("2026-04-16T10-00-00Z.m4a"), b"a")
.await
.unwrap();
tokio::fs::write(
dir.path().join("2026-04-16T10-00-00Z.transcript.txt"),
"text",
)
.await
.unwrap();
seed(dir.path(), "2026-04-16T10-00-00Z", Some("text")).await;
tokio::fs::write(dir.path().join("2026-04-16T10-05-00Z.m4a"), b"b")
.await
.unwrap();
@@ -521,24 +538,8 @@ mod tests {
#[tokio::test]
async fn has_pending_recordings_false_when_all_transcribed() {
let dir = tempdir().unwrap();
tokio::fs::write(dir.path().join("2026-04-16T10-00-00Z.m4a"), b"a")
.await
.unwrap();
tokio::fs::write(
dir.path().join("2026-04-16T10-00-00Z.transcript.txt"),
"text1",
)
.await
.unwrap();
tokio::fs::write(dir.path().join("2026-04-16T10-05-00Z.m4a"), b"b")
.await
.unwrap();
tokio::fs::write(
dir.path().join("2026-04-16T10-05-00Z.transcript.txt"),
"text2",
)
.await
.unwrap();
seed(dir.path(), "2026-04-16T10-00-00Z", Some("text1")).await;
seed(dir.path(), "2026-04-16T10-05-00Z", Some("text2")).await;
assert!(!has_pending_recordings(dir.path()).await);
}