Brummel/doctate
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor recording metadata to JSON sidecar

Browse Source 
Replaces the `.transcript.txt` sidecar with a structured `.json` file
for recording metadata. This change consolidates transcript text,
duration, and other potential metadata into a single, extensible JSON
object.

This also refactors the `TranscriptState` enum to better represent the
on-disk state (absence of file means pending) and the in-memory
representation. The `Transcript` enum now specifically models the
terminal outcomes of the transcriber (`Silent` or `Content`).

The commit includes updates to documentation, data structures, path
handling, and various tests to align with the new metadata format.
This commit is contained in:
Brummel
2026-04-27 12:48:25 +02:00
parent a510c20e75
commit 66b3b7e4c8
20 changed files with 637 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/transcribe/worker.rs
+63 -62
View File
@@ -2,8 +2,8 @@ use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use doctate_common::TranscriptState;
use doctate_common::oneliners::OnelinerState;
use doctate_common::{RecordingMeta, Transcript, TranscriptState};
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;
use tracing::{error, info, warn};
@@ -38,9 +38,13 @@ pub async fn run(
while let Some(job) = rx.recv().await {
let _guard = BusyGuard::new(worker_busy.clone());
let audio_path = job.audio_path;
let transcript_path = audio_path.with_extension("transcript.txt");
let meta_path = paths::recording_meta_path(&audio_path);
if transcript_path.exists() {
// Idempotency gate: presence of `<stem>.json` means the
// transcriber already finished this recording (single atomic
// write at the end of the pipeline). Skip the job, freeing
// Whisper for the next one.
if meta_path.exists() {
continue;
}
@@ -65,11 +69,24 @@ pub async fn run(
}
};
// Persist the container duration as a sidecar so the UI can render it
// without the browser having to HEAD every audio file. Non-fatal — the
// recordings scan has a lazy-backfill path, and transcription is the
// critical workload here.
write_duration_sidecar(&audio_path, remuxed.path()).await;
// Probe the container duration on the remuxed copy (its `moov`
// atom sits at the start, so ffprobe returns without seeking
// to EOF). We carry the value in memory and persist it
// together with the transcript at the end of the pipeline.
// Non-fatal: a probe failure leaves `duration_seconds = None`
// and the recordings scan falls back to a read-only ffprobe
// for the UI render.
let duration_seconds = match ffmpeg::probe_duration_seconds(remuxed.path()).await {
Ok(s) => Some(s),
Err(e) => {
warn!(
audio = %audio_path.display(),
error = %e,
"ffprobe duration failed — meta will record null duration"
);
None
}
};
let text = match whisper::transcribe(
&client,
@@ -106,15 +123,31 @@ pub async fn run(
// the canonical path.
let text = vocab.replace(&text);
if let Err(e) = tokio::fs::write(&transcript_path, &text).await {
error!(transcript = %transcript_path.display(), error = %e, "writing transcript failed");
// Single atomic write: assemble the recording metadata
// (transcript outcome + duration) in memory, then materialise
// it on disk in one rename. Either readers see the previous
// state (no sidecar = pending) or the full new sidecar — never
// a half-written one.
let bytes_logged = text.len();
let transcript = if text.trim().is_empty() {
Transcript::Silent
} else {
Transcript::Content { text }
};
let meta = RecordingMeta {
transcript,
duration_seconds,
};
if let Err(e) = paths::write_recording_meta_atomic(&audio_path, &meta).await {
error!(meta = %meta_path.display(), error = %e, "writing recording meta failed");
continue;
}
info!(
audio = %audio_path.display(),
bytes = text.len(),
"Transcript written"
bytes = bytes_logged,
duration = ?duration_seconds,
"Recording meta written"
);
// Notify the live-update bus so subscribed browsers can refresh the
@@ -185,23 +218,6 @@ async fn mark_failed(audio_path: &Path, events_tx: &EventSender, user_slug: &str
}
}
/// Probe the (remuxed) audio and persist the duration next to the original file
/// as `<ts>.duration.txt`. Probes on the remuxed copy because its `moov` atom
/// sits at the start — ffprobe returns without seeking to EOF.
async fn write_duration_sidecar(audio_path: &Path, probe_path: &Path) {
let secs = match ffmpeg::probe_duration_seconds(probe_path).await {
Ok(s) => s,
Err(e) => {
warn!(audio = %audio_path.display(), error = %e, "ffprobe duration failed — UI will lazy-backfill");
return;
}
};
let sidecar = audio_path.with_extension("duration.txt");
if let Err(e) = tokio::fs::write(&sidecar, secs.to_string()).await {
warn!(sidecar = %sidecar.display(), error = %e, "writing duration sidecar failed");
}
}
/// Regenerate `case_dir/oneliner.json` from **all** `Content` transcripts
/// in the case, joined chronologically. Called after every successful
/// transcript write so later recordings can correct earlier ones (mirror
@@ -408,7 +424,7 @@ pub(crate) async fn all_transcripts_joined(case_dir: &Path) -> Option<String> {
let mut parts: Vec<String> = Vec::with_capacity(m4as.len());
for path in m4as {
if let TranscriptState::Content(text) = paths::read_transcript_state(&path).await {
if let TranscriptState::Content { text } = paths::read_transcript_state(&path).await {
let trimmed = text.trim();
if !trimmed.is_empty() {
parts.push(trimmed.to_string());
@@ -426,18 +442,27 @@ mod tests {
use super::*;
use tempfile::tempdir;
/// Helper: seed an m4a plus an optional transcript sidecar, matching
/// Helper: seed an m4a plus an optional metadata sidecar, matching
/// how the worker lays files out on disk. `all_transcripts_joined`
/// iterates `.m4a` files and reads the sidecar through the shared
/// reader, so bare transcripts without m4as would be ignored.
/// reader, so bare meta files without m4as would be ignored.
///
/// `transcript`: `None` → no sidecar (Pending); `Some(s)` with
/// whitespace-only `s` → `Transcript::Silent`; otherwise
/// `Transcript::Content`.
async fn seed(dir: &Path, stem: &str, transcript: Option<&str>) {
tokio::fs::write(dir.join(format!("{stem}.m4a")), b"audio")
.await
.unwrap();
if let Some(text) = transcript {
tokio::fs::write(dir.join(format!("{stem}.transcript.txt")), text)
.await
.unwrap();
let variant = if text.trim().is_empty() {
Transcript::Silent
} else {
Transcript::Content {
text: text.to_owned(),
}
};
paths::write_recording_meta_sync(dir, stem, variant, None);
}
}
@@ -503,15 +528,7 @@ mod tests {
#[tokio::test]
async fn has_pending_recordings_true_when_m4a_without_transcript() {
let dir = tempdir().unwrap();
tokio::fs::write(dir.path().join("2026-04-16T10-00-00Z.m4a"), b"a")
.await
.unwrap();
tokio::fs::write(
dir.path().join("2026-04-16T10-00-00Z.transcript.txt"),
"text",
)
.await
.unwrap();
seed(dir.path(), "2026-04-16T10-00-00Z", Some("text")).await;
tokio::fs::write(dir.path().join("2026-04-16T10-05-00Z.m4a"), b"b")
.await
.unwrap();
@@ -521,24 +538,8 @@ mod tests {
#[tokio::test]
async fn has_pending_recordings_false_when_all_transcribed() {
let dir = tempdir().unwrap();
tokio::fs::write(dir.path().join("2026-04-16T10-00-00Z.m4a"), b"a")
.await
.unwrap();
tokio::fs::write(
dir.path().join("2026-04-16T10-00-00Z.transcript.txt"),
"text1",
)
.await
.unwrap();
tokio::fs::write(dir.path().join("2026-04-16T10-05-00Z.m4a"), b"b")
.await
.unwrap();
tokio::fs::write(
dir.path().join("2026-04-16T10-05-00Z.transcript.txt"),
"text2",
)
.await
.unwrap();
seed(dir.path(), "2026-04-16T10-00-00Z", Some("text1")).await;
seed(dir.path(), "2026-04-16T10-05-00Z", Some("text2")).await;
assert!(!has_pending_recordings(dir.path()).await);
}