Add loudness data to recording metadata
This commit introduces the `Loudness` struct to store replay-gain data (mean, max, and calculated gain in dB) derived from `ffmpeg`'s `volumedetect`. The `RecordingMeta` struct is updated to include an optional `loudness` field. This allows for consistent playback volume across recordings with varying microphone levels by applying gain in the browser. Additionally, new `server/src/loudness.rs` and `server/src/transcribe/ffmpeg.rs` modules are added. The former defines constants for replay-gain targets and logic to compute gain, while the latter handles audio analysis using `ffmpeg` to extract loudness metrics. The `#[serde(default)]` attribute on the `loudness` field in `RecordingMeta` ensures backward compatibility with older metadata files that do not contain this field. Tests are included to verify round-trip serialization and parsing of older formats.
This commit is contained in:
@@ -51,12 +51,28 @@ pub enum Transcript {
|
||||
Content { text: String },
|
||||
}
|
||||
|
||||
/// Replay-gain data for one recording, derived from a single
|
||||
/// `volumedetect` pass at the end of the transcribe pipeline.
|
||||
///
|
||||
/// `gain_db` is applied browser-side via a Web Audio `GainNode` so
|
||||
/// recordings from clients with very different mic levels (Watch
|
||||
/// ~-38 dB mean, desktop ~-27 dB) play back at a consistent loudness;
|
||||
/// the audio file itself is never modified. `mean_db` and `max_db` are
|
||||
/// persisted alongside so the gain target can be retuned later without
|
||||
/// re-decoding the audio.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct Loudness {
|
||||
pub mean_db: f64,
|
||||
pub max_db: f64,
|
||||
pub gain_db: f64,
|
||||
}
|
||||
|
||||
/// Per-recording metadata persisted as `<stem>.json`.
|
||||
///
|
||||
/// Single atomic write at the end of the transcribe pipeline. The
|
||||
/// presence of this file is the canonical signal "the transcriber
|
||||
/// finished for this recording". Absent file = still pending.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct RecordingMeta {
|
||||
/// Transcriber outcome. See [`Transcript`].
|
||||
pub transcript: Transcript,
|
||||
@@ -65,6 +81,11 @@ pub struct RecordingMeta {
|
||||
/// flows that mint a metadata file without invoking ffprobe stay
|
||||
/// valid; production worker writes always populate this.
|
||||
pub duration_seconds: Option<u32>,
|
||||
/// Replay-gain data. `None` for sidecars written before this field
|
||||
/// existed and for production paths where `volumedetect` failed.
|
||||
/// `#[serde(default)]` keeps older JSON without this field parsable.
|
||||
#[serde(default)]
|
||||
pub loudness: Option<Loudness>,
|
||||
}
|
||||
|
||||
/// Three-way state of a recording's transcript, as seen by the server
|
||||
@@ -145,6 +166,7 @@ mod tests {
|
||||
let meta = RecordingMeta {
|
||||
transcript: Transcript::Silent,
|
||||
duration_seconds: Some(7),
|
||||
loudness: None,
|
||||
};
|
||||
assert_eq!(
|
||||
TranscriptState::from_meta(Some(&meta)),
|
||||
@@ -159,6 +181,7 @@ mod tests {
|
||||
text: "Hallo".into(),
|
||||
},
|
||||
duration_seconds: None,
|
||||
loudness: None,
|
||||
};
|
||||
assert_eq!(
|
||||
TranscriptState::from_meta(Some(&meta)),
|
||||
@@ -207,6 +230,7 @@ mod tests {
|
||||
text: "Befund.".into(),
|
||||
},
|
||||
duration_seconds: Some(42),
|
||||
loudness: None,
|
||||
};
|
||||
let json = serde_json::to_string(&meta).unwrap();
|
||||
let back: RecordingMeta = serde_json::from_str(&json).unwrap();
|
||||
@@ -218,10 +242,47 @@ mod tests {
|
||||
let meta = RecordingMeta {
|
||||
transcript: Transcript::Silent,
|
||||
duration_seconds: None,
|
||||
loudness: None,
|
||||
};
|
||||
let json = serde_json::to_string(&meta).unwrap();
|
||||
assert!(json.contains(r#""duration_seconds":null"#));
|
||||
let back: RecordingMeta = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(back, meta);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recording_meta_roundtrip_with_loudness() {
|
||||
let meta = RecordingMeta {
|
||||
transcript: Transcript::Content {
|
||||
text: "Befund.".into(),
|
||||
},
|
||||
duration_seconds: Some(42),
|
||||
loudness: Some(Loudness {
|
||||
mean_db: -27.3,
|
||||
max_db: -3.1,
|
||||
gain_db: 2.1,
|
||||
}),
|
||||
};
|
||||
let json = serde_json::to_string(&meta).unwrap();
|
||||
let back: RecordingMeta = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(back, meta);
|
||||
}
|
||||
|
||||
/// Backwards-compat: a sidecar written before `loudness` existed
|
||||
/// must still parse, with the field defaulting to `None`. This is
|
||||
/// what `#[serde(default)]` on the field guarantees and is the only
|
||||
/// reason we don't need a migration step for pre-loudness recordings.
|
||||
#[test]
|
||||
fn old_meta_without_loudness_parses_as_none() {
|
||||
let json = r#"{"transcript":{"state":"silent"},"duration_seconds":7}"#;
|
||||
let meta: RecordingMeta = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(
|
||||
meta,
|
||||
RecordingMeta {
|
||||
transcript: Transcript::Silent,
|
||||
duration_seconds: Some(7),
|
||||
loudness: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ pub mod csrf;
|
||||
pub mod error;
|
||||
pub mod events;
|
||||
pub mod gazetteer;
|
||||
pub mod loudness;
|
||||
pub mod magic_link;
|
||||
pub mod models;
|
||||
pub mod oneliner_locks;
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
//! Replay-gain target curve, applied browser-side via Web Audio.
|
||||
//!
|
||||
//! Goal: equalize playback loudness across recordings from clients
|
||||
//! with very different microphone levels (Watch ~-38 dB mean, desktop
|
||||
//! ~-27 dB) without modifying the audio file. The server computes one
|
||||
//! non-negative gain value per recording from a `volumedetect` pass;
|
||||
//! the browser applies it through a `GainNode`.
|
||||
|
||||
/// Target mean volume in dBFS. Recordings below this get amplified
|
||||
/// up to this level; recordings already at or above it get gain 0.
|
||||
/// Matches the EBU R128 broadcast loudness ballpark for speech and
|
||||
/// was validated against the 2026-04-23 PoC.
|
||||
pub const TARGET_MEAN_DB: f64 = -16.0;
|
||||
|
||||
/// Peak ceiling in dBFS. Caps amplification so the loudest sample
|
||||
/// never exceeds this — leaves a 1 dB headroom against digital
|
||||
/// clipping. Combined with the target-mean check, the chosen gain is
|
||||
/// the minimum of the two budgets.
|
||||
pub const PEAK_CEILING_DB: f64 = -1.0;
|
||||
|
||||
/// Compute the playback-gain in dB for a recording with the given
|
||||
/// `volumedetect` measurements. Always non-negative — we never
|
||||
/// attenuate; if the recording is already at or above the target it
|
||||
/// gets gain 0 dB.
|
||||
///
|
||||
/// **Caller contract:** both inputs must be finite (`is_finite()`).
|
||||
/// `f64::NEG_INFINITY` (silent file) propagates to `+Inf` through the
|
||||
/// arithmetic and would make `serde_json` refuse to serialize the
|
||||
/// resulting `Loudness` struct. The worker gates on finiteness and
|
||||
/// stores `loudness: None` for silent recordings instead.
|
||||
pub fn compute_gain_db(mean_db: f64, max_db: f64) -> f64 {
|
||||
let to_target = TARGET_MEAN_DB - mean_db;
|
||||
let headroom = PEAK_CEILING_DB - max_db;
|
||||
to_target.min(headroom).max(0.0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A very dynamic source: low average level (mean=-30) but loud
|
||||
/// peaks (max=-3). The headroom budget caps the gain even though
|
||||
/// the average alone would allow much more boost.
|
||||
#[test]
|
||||
fn headroom_limits_gain() {
|
||||
// to_target = -16 - (-30) = +14
|
||||
// headroom = -1 - (-3) = +2 ← tighter
|
||||
assert_eq!(compute_gain_db(-30.0, -3.0), 2.0);
|
||||
}
|
||||
|
||||
/// A quiet, well-controlled source: low average (mean=-22),
|
||||
/// moderate peaks (max=-15). Plenty of headroom — the target
|
||||
/// budget is the limiting factor.
|
||||
#[test]
|
||||
fn target_limits_gain() {
|
||||
// to_target = -16 - (-22) = +6 ← tighter
|
||||
// headroom = -1 - (-15) = +14
|
||||
assert_eq!(compute_gain_db(-22.0, -15.0), 6.0);
|
||||
}
|
||||
|
||||
/// Already-loud source (mean=-12, well above target). Both
|
||||
/// budgets contradict each other, but the `max(0.0)` floor makes
|
||||
/// the result 0 — we never attenuate.
|
||||
#[test]
|
||||
fn clamps_to_zero_when_already_loud() {
|
||||
// to_target = -16 - (-12) = -4 (would attenuate)
|
||||
// headroom = -1 - (-2) = +1
|
||||
// min = -4
|
||||
// max(0) = 0
|
||||
assert_eq!(compute_gain_db(-12.0, -2.0), 0.0);
|
||||
}
|
||||
}
|
||||
@@ -240,6 +240,7 @@ pub(crate) fn write_recording_meta_sync(
|
||||
let meta = doctate_common::RecordingMeta {
|
||||
transcript,
|
||||
duration_seconds,
|
||||
loudness: None,
|
||||
};
|
||||
let bytes = serde_json::to_vec(&meta).expect("RecordingMeta serializes infallibly");
|
||||
std::fs::write(
|
||||
|
||||
@@ -109,3 +109,175 @@ pub async fn probe_duration_seconds(path: &Path) -> Result<u32, FfmpegError> {
|
||||
}
|
||||
Ok(secs.round() as u32)
|
||||
}
|
||||
|
||||
/// Result of one `ffmpeg -af volumedetect` pass on an audio file:
|
||||
/// duration plus mean and peak volume in a single decode.
|
||||
///
|
||||
/// `mean_db` and `max_db` may be `f64::NEG_INFINITY` for fully silent
|
||||
/// (zero-sample) input — `f64::from_str` accepts `"-inf"` natively.
|
||||
/// Callers that build a `Loudness` value must gate on `is_finite()`
|
||||
/// to avoid feeding `+Inf` gain into Web Audio.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct AudioAnalysis {
|
||||
pub duration_seconds: u32,
|
||||
pub mean_db: f64,
|
||||
pub max_db: f64,
|
||||
}
|
||||
|
||||
/// Run `ffmpeg ... -af volumedetect -f null -` against `path` and
|
||||
/// extract duration plus mean/peak volume from its stderr in a single
|
||||
/// decode pass.
|
||||
///
|
||||
/// Decode-bound: O(audio length) wall-time, ~100-500 ms for typical
|
||||
/// 10-60 s dictations on one CPU core. Worker code overlaps this with
|
||||
/// Whisper via `tokio::join!`, so on the pipeline wallclock the cost
|
||||
/// is effectively hidden behind the Whisper call.
|
||||
pub async fn analyze_audio(path: &Path) -> Result<AudioAnalysis, FfmpegError> {
|
||||
let output = Command::new("ffmpeg")
|
||||
.arg("-hide_banner")
|
||||
.arg("-nostats")
|
||||
.arg("-i")
|
||||
.arg(path)
|
||||
.arg("-af")
|
||||
.arg("volumedetect")
|
||||
.arg("-f")
|
||||
.arg("null")
|
||||
.arg("-")
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.output()
|
||||
.await
|
||||
.map_err(FfmpegError::Spawn)?;
|
||||
|
||||
if !output.status.success() {
|
||||
return Err(FfmpegError::NonZeroExit {
|
||||
code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
|
||||
});
|
||||
}
|
||||
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
parse_analysis(&stderr)
|
||||
}
|
||||
|
||||
/// Pure-string parser split out from [`analyze_audio`] so unit tests
|
||||
/// can target the parsing rules without spawning a real ffmpeg.
|
||||
fn parse_analysis(stderr: &str) -> Result<AudioAnalysis, FfmpegError> {
|
||||
let duration_seconds = parse_duration_seconds(stderr)
|
||||
.ok_or_else(|| FfmpegError::Parse("missing or unparseable Duration line".into()))?;
|
||||
let mean_db = parse_volume_db(stderr, "mean_volume")
|
||||
.ok_or_else(|| FfmpegError::Parse("missing or unparseable mean_volume line".into()))?;
|
||||
let max_db = parse_volume_db(stderr, "max_volume")
|
||||
.ok_or_else(|| FfmpegError::Parse("missing or unparseable max_volume line".into()))?;
|
||||
Ok(AudioAnalysis {
|
||||
duration_seconds,
|
||||
mean_db,
|
||||
max_db,
|
||||
})
|
||||
}
|
||||
|
||||
/// Locate the first `Duration: HH:MM:SS.cs, ...` line in ffmpeg's
|
||||
/// stderr and convert it to whole rounded seconds.
|
||||
fn parse_duration_seconds(stderr: &str) -> Option<u32> {
|
||||
let line = stderr
|
||||
.lines()
|
||||
.find(|l| l.trim_start().starts_with("Duration: "))?;
|
||||
let timecode = line
|
||||
.trim_start()
|
||||
.strip_prefix("Duration: ")?
|
||||
.split(',')
|
||||
.next()?
|
||||
.trim();
|
||||
let mut parts = timecode.split(':');
|
||||
let hours: f64 = parts.next()?.parse().ok()?;
|
||||
let minutes: f64 = parts.next()?.parse().ok()?;
|
||||
let seconds: f64 = parts.next()?.parse().ok()?;
|
||||
let total = hours * 3600.0 + minutes * 60.0 + seconds;
|
||||
if !total.is_finite() || total < 0.0 {
|
||||
return None;
|
||||
}
|
||||
Some(total.round() as u32)
|
||||
}
|
||||
|
||||
/// Find a `<key>: <value> dB` entry in stderr and return `<value>` as
|
||||
/// `f64`. `f64::from_str` accepts `-inf` / `inf` natively, which is
|
||||
/// what we need for fully silent input.
|
||||
fn parse_volume_db(stderr: &str, key: &str) -> Option<f64> {
|
||||
let needle = format!("{key}: ");
|
||||
let line = stderr.lines().find(|l| l.contains(&needle))?;
|
||||
let after = &line[line.find(&needle)? + needle.len()..];
|
||||
let token = after.split_whitespace().next()?;
|
||||
token.parse().ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Realistic ffmpeg stderr for a successful volumedetect pass on a
|
||||
/// ~42-second mono dictation, trimmed to the lines our parser cares
|
||||
/// about. Format mirrors a real run; lifted verbatim modulo the
|
||||
/// memory-address tokens.
|
||||
const REALISTIC_STDERR: &str = "\
|
||||
Input #0, mov,mp4,m4a,3gp,3g2,mj2, from '/tmp/x.m4a':
|
||||
Metadata:
|
||||
major_brand : mp42
|
||||
Duration: 00:00:42.31, start: 0.000000, bitrate: 64 kb/s
|
||||
Stream #0:0(und): Audio: aac (LC) (mp4a / 0x6134706D), 16000 Hz, mono, fltp, 64 kb/s
|
||||
[Parsed_volumedetect_0 @ 0x55] mean_volume: -27.3 dB
|
||||
[Parsed_volumedetect_0 @ 0x55] max_volume: -3.1 dB
|
||||
[Parsed_volumedetect_0 @ 0x55] histogram_0db: 5
|
||||
";
|
||||
|
||||
#[test]
|
||||
fn parse_realistic_stderr() {
|
||||
let got = parse_analysis(REALISTIC_STDERR).expect("Ok");
|
||||
assert_eq!(
|
||||
got,
|
||||
AudioAnalysis {
|
||||
duration_seconds: 42,
|
||||
mean_db: -27.3,
|
||||
max_db: -3.1,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn silent_file_yields_neg_infinity() {
|
||||
let stderr = "\
|
||||
Input #0, ...:
|
||||
Duration: 00:00:05.00, start: 0.000000, bitrate: 64 kb/s
|
||||
[Parsed_volumedetect_0 @ 0x55] mean_volume: -inf dB
|
||||
[Parsed_volumedetect_0 @ 0x55] max_volume: -inf dB
|
||||
";
|
||||
let got = parse_analysis(stderr).expect("Ok");
|
||||
assert_eq!(got.duration_seconds, 5);
|
||||
assert!(got.mean_db.is_infinite() && got.mean_db.is_sign_negative());
|
||||
assert!(got.max_db.is_infinite() && got.max_db.is_sign_negative());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_duration_is_parse_error() {
|
||||
let stderr = "\
|
||||
[Parsed_volumedetect_0 @ 0x55] mean_volume: -27.3 dB
|
||||
[Parsed_volumedetect_0 @ 0x55] max_volume: -3.1 dB
|
||||
";
|
||||
match parse_analysis(stderr) {
|
||||
Err(FfmpegError::Parse(_)) => {}
|
||||
other => panic!("expected Parse error, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_mean_volume_is_parse_error() {
|
||||
let stderr = "\
|
||||
Duration: 00:00:42.31, start: 0.000000, bitrate: 64 kb/s
|
||||
[Parsed_volumedetect_0 @ 0x55] max_volume: -3.1 dB
|
||||
";
|
||||
match parse_analysis(stderr) {
|
||||
Err(FfmpegError::Parse(_)) => {}
|
||||
other => panic!("expected Parse error, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,6 +137,7 @@ pub async fn run(
|
||||
let meta = RecordingMeta {
|
||||
transcript,
|
||||
duration_seconds,
|
||||
loudness: None,
|
||||
};
|
||||
if let Err(e) = paths::write_recording_meta_atomic(&audio_path, &meta).await {
|
||||
error!(meta = %meta_path.display(), error = %e, "writing recording meta failed");
|
||||
|
||||
@@ -68,6 +68,7 @@ pub fn seed_recording_meta(
|
||||
let meta = RecordingMeta {
|
||||
transcript,
|
||||
duration_seconds,
|
||||
loudness: None,
|
||||
};
|
||||
let bytes = serde_json::to_vec(&meta).unwrap();
|
||||
std::fs::write(
|
||||
|
||||
Reference in New Issue
Block a user