From 427a28ac6b7fc532544f7a7bca822826cfd5cad2 Mon Sep 17 00:00:00 2001 From: Brummel Date: Mon, 27 Apr 2026 16:11:02 +0200 Subject: [PATCH] Add loudness data to recording metadata This commit introduces the `Loudness` struct to store replay-gain data (mean, max, and calculated gain in dB) derived from `ffmpeg`'s `volumedetect`. The `RecordingMeta` struct is updated to include an optional `loudness` field. This allows for consistent playback volume across recordings with varying microphone levels by applying gain in the browser. Additionally, new `server/src/loudness.rs` and `server/src/transcribe/ffmpeg.rs` modules are added. The former defines constants for replay-gain targets and logic to compute gain, while the latter handles audio analysis using `ffmpeg` to extract loudness metrics. The `#[serde(default)]` attribute on the `loudness` field in `RecordingMeta` ensures backward compatibility with older metadata files that do not contain this field. Tests are included to verify round-trip serialization and parsing of older formats. --- doctate-common/src/recordings.rs | 63 ++++++++++- server/src/lib.rs | 1 + server/src/loudness.rs | 72 +++++++++++++ server/src/paths.rs | 1 + server/src/transcribe/ffmpeg.rs | 172 +++++++++++++++++++++++++++++++ server/src/transcribe/worker.rs | 1 + server/tests/common/seed.rs | 1 + 7 files changed, 310 insertions(+), 1 deletion(-) create mode 100644 server/src/loudness.rs diff --git a/doctate-common/src/recordings.rs b/doctate-common/src/recordings.rs index 1add151..3569e4f 100644 --- a/doctate-common/src/recordings.rs +++ b/doctate-common/src/recordings.rs @@ -51,12 +51,28 @@ pub enum Transcript { Content { text: String }, } +/// Replay-gain data for one recording, derived from a single +/// `volumedetect` pass at the end of the transcribe pipeline. +/// +/// `gain_db` is applied browser-side via a Web Audio `GainNode` so +/// recordings from clients with very different mic levels (Watch +/// ~-38 dB mean, desktop ~-27 dB) play back at a consistent loudness; +/// the audio file itself is never modified. `mean_db` and `max_db` are +/// persisted alongside so the gain target can be retuned later without +/// re-decoding the audio. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Loudness { + pub mean_db: f64, + pub max_db: f64, + pub gain_db: f64, +} + /// Per-recording metadata persisted as `.json`. /// /// Single atomic write at the end of the transcribe pipeline. The /// presence of this file is the canonical signal "the transcriber /// finished for this recording". Absent file = still pending. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct RecordingMeta { /// Transcriber outcome. See [`Transcript`]. pub transcript: Transcript, @@ -65,6 +81,11 @@ pub struct RecordingMeta { /// flows that mint a metadata file without invoking ffprobe stay /// valid; production worker writes always populate this. pub duration_seconds: Option, + /// Replay-gain data. `None` for sidecars written before this field + /// existed and for production paths where `volumedetect` failed. + /// `#[serde(default)]` keeps older JSON without this field parsable. + #[serde(default)] + pub loudness: Option, } /// Three-way state of a recording's transcript, as seen by the server @@ -145,6 +166,7 @@ mod tests { let meta = RecordingMeta { transcript: Transcript::Silent, duration_seconds: Some(7), + loudness: None, }; assert_eq!( TranscriptState::from_meta(Some(&meta)), @@ -159,6 +181,7 @@ mod tests { text: "Hallo".into(), }, duration_seconds: None, + loudness: None, }; assert_eq!( TranscriptState::from_meta(Some(&meta)), @@ -207,6 +230,7 @@ mod tests { text: "Befund.".into(), }, duration_seconds: Some(42), + loudness: None, }; let json = serde_json::to_string(&meta).unwrap(); let back: RecordingMeta = serde_json::from_str(&json).unwrap(); @@ -218,10 +242,47 @@ mod tests { let meta = RecordingMeta { transcript: Transcript::Silent, duration_seconds: None, + loudness: None, }; let json = serde_json::to_string(&meta).unwrap(); assert!(json.contains(r#""duration_seconds":null"#)); let back: RecordingMeta = serde_json::from_str(&json).unwrap(); assert_eq!(back, meta); } + + #[test] + fn recording_meta_roundtrip_with_loudness() { + let meta = RecordingMeta { + transcript: Transcript::Content { + text: "Befund.".into(), + }, + duration_seconds: Some(42), + loudness: Some(Loudness { + mean_db: -27.3, + max_db: -3.1, + gain_db: 2.1, + }), + }; + let json = serde_json::to_string(&meta).unwrap(); + let back: RecordingMeta = serde_json::from_str(&json).unwrap(); + assert_eq!(back, meta); + } + + /// Backwards-compat: a sidecar written before `loudness` existed + /// must still parse, with the field defaulting to `None`. This is + /// what `#[serde(default)]` on the field guarantees and is the only + /// reason we don't need a migration step for pre-loudness recordings. + #[test] + fn old_meta_without_loudness_parses_as_none() { + let json = r#"{"transcript":{"state":"silent"},"duration_seconds":7}"#; + let meta: RecordingMeta = serde_json::from_str(json).unwrap(); + assert_eq!( + meta, + RecordingMeta { + transcript: Transcript::Silent, + duration_seconds: Some(7), + loudness: None, + } + ); + } } diff --git a/server/src/lib.rs b/server/src/lib.rs index 7e5927d..26a1915 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -6,6 +6,7 @@ pub mod csrf; pub mod error; pub mod events; pub mod gazetteer; +pub mod loudness; pub mod magic_link; pub mod models; pub mod oneliner_locks; diff --git a/server/src/loudness.rs b/server/src/loudness.rs new file mode 100644 index 0000000..8f3c5bd --- /dev/null +++ b/server/src/loudness.rs @@ -0,0 +1,72 @@ +//! Replay-gain target curve, applied browser-side via Web Audio. +//! +//! Goal: equalize playback loudness across recordings from clients +//! with very different microphone levels (Watch ~-38 dB mean, desktop +//! ~-27 dB) without modifying the audio file. The server computes one +//! non-negative gain value per recording from a `volumedetect` pass; +//! the browser applies it through a `GainNode`. + +/// Target mean volume in dBFS. Recordings below this get amplified +/// up to this level; recordings already at or above it get gain 0. +/// Matches the EBU R128 broadcast loudness ballpark for speech and +/// was validated against the 2026-04-23 PoC. +pub const TARGET_MEAN_DB: f64 = -16.0; + +/// Peak ceiling in dBFS. Caps amplification so the loudest sample +/// never exceeds this — leaves a 1 dB headroom against digital +/// clipping. Combined with the target-mean check, the chosen gain is +/// the minimum of the two budgets. +pub const PEAK_CEILING_DB: f64 = -1.0; + +/// Compute the playback-gain in dB for a recording with the given +/// `volumedetect` measurements. Always non-negative — we never +/// attenuate; if the recording is already at or above the target it +/// gets gain 0 dB. +/// +/// **Caller contract:** both inputs must be finite (`is_finite()`). +/// `f64::NEG_INFINITY` (silent file) propagates to `+Inf` through the +/// arithmetic and would make `serde_json` refuse to serialize the +/// resulting `Loudness` struct. The worker gates on finiteness and +/// stores `loudness: None` for silent recordings instead. +pub fn compute_gain_db(mean_db: f64, max_db: f64) -> f64 { + let to_target = TARGET_MEAN_DB - mean_db; + let headroom = PEAK_CEILING_DB - max_db; + to_target.min(headroom).max(0.0) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A very dynamic source: low average level (mean=-30) but loud + /// peaks (max=-3). The headroom budget caps the gain even though + /// the average alone would allow much more boost. + #[test] + fn headroom_limits_gain() { + // to_target = -16 - (-30) = +14 + // headroom = -1 - (-3) = +2 ← tighter + assert_eq!(compute_gain_db(-30.0, -3.0), 2.0); + } + + /// A quiet, well-controlled source: low average (mean=-22), + /// moderate peaks (max=-15). Plenty of headroom — the target + /// budget is the limiting factor. + #[test] + fn target_limits_gain() { + // to_target = -16 - (-22) = +6 ← tighter + // headroom = -1 - (-15) = +14 + assert_eq!(compute_gain_db(-22.0, -15.0), 6.0); + } + + /// Already-loud source (mean=-12, well above target). Both + /// budgets contradict each other, but the `max(0.0)` floor makes + /// the result 0 — we never attenuate. + #[test] + fn clamps_to_zero_when_already_loud() { + // to_target = -16 - (-12) = -4 (would attenuate) + // headroom = -1 - (-2) = +1 + // min = -4 + // max(0) = 0 + assert_eq!(compute_gain_db(-12.0, -2.0), 0.0); + } +} diff --git a/server/src/paths.rs b/server/src/paths.rs index cbbbb8f..73892b9 100644 --- a/server/src/paths.rs +++ b/server/src/paths.rs @@ -240,6 +240,7 @@ pub(crate) fn write_recording_meta_sync( let meta = doctate_common::RecordingMeta { transcript, duration_seconds, + loudness: None, }; let bytes = serde_json::to_vec(&meta).expect("RecordingMeta serializes infallibly"); std::fs::write( diff --git a/server/src/transcribe/ffmpeg.rs b/server/src/transcribe/ffmpeg.rs index 707846d..d0f04f2 100644 --- a/server/src/transcribe/ffmpeg.rs +++ b/server/src/transcribe/ffmpeg.rs @@ -109,3 +109,175 @@ pub async fn probe_duration_seconds(path: &Path) -> Result { } Ok(secs.round() as u32) } + +/// Result of one `ffmpeg -af volumedetect` pass on an audio file: +/// duration plus mean and peak volume in a single decode. +/// +/// `mean_db` and `max_db` may be `f64::NEG_INFINITY` for fully silent +/// (zero-sample) input — `f64::from_str` accepts `"-inf"` natively. +/// Callers that build a `Loudness` value must gate on `is_finite()` +/// to avoid feeding `+Inf` gain into Web Audio. +#[derive(Debug, Clone, PartialEq)] +pub struct AudioAnalysis { + pub duration_seconds: u32, + pub mean_db: f64, + pub max_db: f64, +} + +/// Run `ffmpeg ... -af volumedetect -f null -` against `path` and +/// extract duration plus mean/peak volume from its stderr in a single +/// decode pass. +/// +/// Decode-bound: O(audio length) wall-time, ~100-500 ms for typical +/// 10-60 s dictations on one CPU core. Worker code overlaps this with +/// Whisper via `tokio::join!`, so on the pipeline wallclock the cost +/// is effectively hidden behind the Whisper call. +pub async fn analyze_audio(path: &Path) -> Result { + let output = Command::new("ffmpeg") + .arg("-hide_banner") + .arg("-nostats") + .arg("-i") + .arg(path) + .arg("-af") + .arg("volumedetect") + .arg("-f") + .arg("null") + .arg("-") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(FfmpegError::Spawn)?; + + if !output.status.success() { + return Err(FfmpegError::NonZeroExit { + code: output.status.code(), + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + }); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + parse_analysis(&stderr) +} + +/// Pure-string parser split out from [`analyze_audio`] so unit tests +/// can target the parsing rules without spawning a real ffmpeg. +fn parse_analysis(stderr: &str) -> Result { + let duration_seconds = parse_duration_seconds(stderr) + .ok_or_else(|| FfmpegError::Parse("missing or unparseable Duration line".into()))?; + let mean_db = parse_volume_db(stderr, "mean_volume") + .ok_or_else(|| FfmpegError::Parse("missing or unparseable mean_volume line".into()))?; + let max_db = parse_volume_db(stderr, "max_volume") + .ok_or_else(|| FfmpegError::Parse("missing or unparseable max_volume line".into()))?; + Ok(AudioAnalysis { + duration_seconds, + mean_db, + max_db, + }) +} + +/// Locate the first `Duration: HH:MM:SS.cs, ...` line in ffmpeg's +/// stderr and convert it to whole rounded seconds. +fn parse_duration_seconds(stderr: &str) -> Option { + let line = stderr + .lines() + .find(|l| l.trim_start().starts_with("Duration: "))?; + let timecode = line + .trim_start() + .strip_prefix("Duration: ")? + .split(',') + .next()? + .trim(); + let mut parts = timecode.split(':'); + let hours: f64 = parts.next()?.parse().ok()?; + let minutes: f64 = parts.next()?.parse().ok()?; + let seconds: f64 = parts.next()?.parse().ok()?; + let total = hours * 3600.0 + minutes * 60.0 + seconds; + if !total.is_finite() || total < 0.0 { + return None; + } + Some(total.round() as u32) +} + +/// Find a `: dB` entry in stderr and return `` as +/// `f64`. `f64::from_str` accepts `-inf` / `inf` natively, which is +/// what we need for fully silent input. +fn parse_volume_db(stderr: &str, key: &str) -> Option { + let needle = format!("{key}: "); + let line = stderr.lines().find(|l| l.contains(&needle))?; + let after = &line[line.find(&needle)? + needle.len()..]; + let token = after.split_whitespace().next()?; + token.parse().ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Realistic ffmpeg stderr for a successful volumedetect pass on a + /// ~42-second mono dictation, trimmed to the lines our parser cares + /// about. Format mirrors a real run; lifted verbatim modulo the + /// memory-address tokens. + const REALISTIC_STDERR: &str = "\ +Input #0, mov,mp4,m4a,3gp,3g2,mj2, from '/tmp/x.m4a': + Metadata: + major_brand : mp42 + Duration: 00:00:42.31, start: 0.000000, bitrate: 64 kb/s + Stream #0:0(und): Audio: aac (LC) (mp4a / 0x6134706D), 16000 Hz, mono, fltp, 64 kb/s +[Parsed_volumedetect_0 @ 0x55] mean_volume: -27.3 dB +[Parsed_volumedetect_0 @ 0x55] max_volume: -3.1 dB +[Parsed_volumedetect_0 @ 0x55] histogram_0db: 5 +"; + + #[test] + fn parse_realistic_stderr() { + let got = parse_analysis(REALISTIC_STDERR).expect("Ok"); + assert_eq!( + got, + AudioAnalysis { + duration_seconds: 42, + mean_db: -27.3, + max_db: -3.1, + } + ); + } + + #[test] + fn silent_file_yields_neg_infinity() { + let stderr = "\ +Input #0, ...: + Duration: 00:00:05.00, start: 0.000000, bitrate: 64 kb/s +[Parsed_volumedetect_0 @ 0x55] mean_volume: -inf dB +[Parsed_volumedetect_0 @ 0x55] max_volume: -inf dB +"; + let got = parse_analysis(stderr).expect("Ok"); + assert_eq!(got.duration_seconds, 5); + assert!(got.mean_db.is_infinite() && got.mean_db.is_sign_negative()); + assert!(got.max_db.is_infinite() && got.max_db.is_sign_negative()); + } + + #[test] + fn missing_duration_is_parse_error() { + let stderr = "\ +[Parsed_volumedetect_0 @ 0x55] mean_volume: -27.3 dB +[Parsed_volumedetect_0 @ 0x55] max_volume: -3.1 dB +"; + match parse_analysis(stderr) { + Err(FfmpegError::Parse(_)) => {} + other => panic!("expected Parse error, got {other:?}"), + } + } + + #[test] + fn missing_mean_volume_is_parse_error() { + let stderr = "\ + Duration: 00:00:42.31, start: 0.000000, bitrate: 64 kb/s +[Parsed_volumedetect_0 @ 0x55] max_volume: -3.1 dB +"; + match parse_analysis(stderr) { + Err(FfmpegError::Parse(_)) => {} + other => panic!("expected Parse error, got {other:?}"), + } + } +} diff --git a/server/src/transcribe/worker.rs b/server/src/transcribe/worker.rs index af78498..96e587b 100644 --- a/server/src/transcribe/worker.rs +++ b/server/src/transcribe/worker.rs @@ -137,6 +137,7 @@ pub async fn run( let meta = RecordingMeta { transcript, duration_seconds, + loudness: None, }; if let Err(e) = paths::write_recording_meta_atomic(&audio_path, &meta).await { error!(meta = %meta_path.display(), error = %e, "writing recording meta failed"); diff --git a/server/tests/common/seed.rs b/server/tests/common/seed.rs index 18af55b..da95ab4 100644 --- a/server/tests/common/seed.rs +++ b/server/tests/common/seed.rs @@ -68,6 +68,7 @@ pub fn seed_recording_meta( let meta = RecordingMeta { transcript, duration_seconds, + loudness: None, }; let bytes = serde_json::to_vec(&meta).unwrap(); std::fs::write(