cab71e6a26
This commit introduces a new startup cleanup routine for the Doctate client. The cleanup process reaps stale markers and orphaned audio files to adhere to data minimization principles and maintain client efficiency. Specifically, it performs the following actions: - Removes markers that are older than 72 hours and have been confirmed by the server. This ensures that old, irrelevant metadata is purged. - Cleans up orphaned audio files (e.g., `.m4a` without corresponding `.meta.json` or vice-versa) that are older than 24 hours. This prevents the accumulation of audio data that will never be uploaded, as such files are silently ignored by the uploader. - Removes any temporary files (`.tmp`) left over from interrupted writes, as these are considered invalid at startup. The `filetime` crate is added as a dependency to facilitate setting modification times for testing purposes.
220 lines
8.6 KiB
Rust
220 lines
8.6 KiB
Rust
//! Orphan reaping for the client's pending-upload directory.
|
|
//!
|
|
//! The uploader happy-path deletes `.m4a` + `.meta.json` pairs after
|
|
//! ACK. A crash between recorder-stop and sidecar-write, or a crash
|
|
//! between ACK and the two `remove_file` calls, can leave **orphans**
|
|
//! behind:
|
|
//!
|
|
//! - `*.m4a` without `*.meta.json` (or vice versa): the uploader's
|
|
//! `scan_pending_dir` silently skips these on next launch, so they
|
|
//! would accumulate indefinitely without explicit cleanup.
|
|
//! - `*.json.tmp` / `*.m4a.tmp` leftovers from interrupted atomic
|
|
//! writes: pure garbage, always safe to remove.
|
|
//!
|
|
//! Orphan audio is sensitive — clients must not hoard patient
|
|
//! recordings that never made it to the server. This module reaps them
|
|
//! on a caller-supplied age threshold. The caller passes an `age_cutoff:
|
|
//! SystemTime`; everything whose mtime is strictly older is deleted.
|
|
//! `.tmp` leftovers bypass the cutoff (they are never legitimate work
|
|
//! in progress at app startup — the lock file guarantees exclusivity).
|
|
|
|
use std::path::Path;
|
|
use std::time::SystemTime;
|
|
|
|
use tracing::{info, warn};
|
|
|
|
/// Reap orphan artefacts in `pending_dir`. Returns the number of files
|
|
/// deleted. Missing directory is treated as "nothing to do".
|
|
pub async fn cleanup_orphan_audio(pending_dir: &Path, age_cutoff: SystemTime) -> usize {
|
|
let Ok(mut entries) = tokio::fs::read_dir(pending_dir).await else {
|
|
return 0;
|
|
};
|
|
|
|
// First pass: collect filenames so we can answer "does the sibling
|
|
// exist?" without doing N separate stat calls.
|
|
let mut names: Vec<String> = Vec::new();
|
|
while let Ok(Some(e)) = entries.next_entry().await {
|
|
if let Some(n) = e.file_name().to_str() {
|
|
names.push(n.to_owned());
|
|
}
|
|
}
|
|
let has_name = |n: &str| names.iter().any(|x| x == n);
|
|
|
|
let mut deleted = 0usize;
|
|
for name in &names {
|
|
let path = pending_dir.join(name);
|
|
|
|
// Atomic-write leftovers: always delete, no age check. A running
|
|
// process would hold the single-instance lock, so `.tmp` at
|
|
// startup is by definition stale.
|
|
if name.ends_with(".m4a.tmp") || name.ends_with(".meta.json.tmp") {
|
|
if let Err(e) = tokio::fs::remove_file(&path).await {
|
|
warn!(path = %path.display(), error = %e, "failed to remove tmp leftover");
|
|
} else {
|
|
info!(path = %path.display(), "tmp leftover removed");
|
|
deleted += 1;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
// Candidate: orphan `.m4a` — only delete if older than cutoff.
|
|
if let Some(stem) = name.strip_suffix(".m4a") {
|
|
let sibling = format!("{stem}.meta.json");
|
|
if has_name(&sibling) {
|
|
continue; // paired, leave for the uploader
|
|
}
|
|
if mtime_older_than(&path, age_cutoff).await {
|
|
if let Err(e) = tokio::fs::remove_file(&path).await {
|
|
warn!(path = %path.display(), error = %e, "failed to remove orphan m4a");
|
|
} else {
|
|
info!(path = %path.display(), "orphan m4a removed (no sidecar)");
|
|
deleted += 1;
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
|
|
// Candidate: orphan sidecar — only delete if older than cutoff.
|
|
if let Some(stem) = name.strip_suffix(".meta.json") {
|
|
let sibling = format!("{stem}.m4a");
|
|
if has_name(&sibling) {
|
|
continue;
|
|
}
|
|
if mtime_older_than(&path, age_cutoff).await {
|
|
if let Err(e) = tokio::fs::remove_file(&path).await {
|
|
warn!(path = %path.display(), error = %e, "failed to remove orphan sidecar");
|
|
} else {
|
|
info!(path = %path.display(), "orphan sidecar removed (no m4a)");
|
|
deleted += 1;
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
}
|
|
|
|
deleted
|
|
}
|
|
|
|
async fn mtime_older_than(path: &Path, cutoff: SystemTime) -> bool {
|
|
match tokio::fs::metadata(path).await {
|
|
Ok(meta) => match meta.modified() {
|
|
Ok(mtime) => mtime < cutoff,
|
|
Err(_) => false,
|
|
},
|
|
Err(_) => false,
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::time::Duration;
|
|
use tempfile::TempDir;
|
|
|
|
async fn touch_with_mtime(path: &Path, mtime: SystemTime) {
|
|
tokio::fs::write(path, b"x").await.unwrap();
|
|
filetime::set_file_mtime(path, filetime::FileTime::from_system_time(mtime)).unwrap();
|
|
}
|
|
|
|
fn now() -> SystemTime {
|
|
SystemTime::now()
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn missing_directory_returns_zero() {
|
|
let tmp = TempDir::new().unwrap();
|
|
let n = cleanup_orphan_audio(&tmp.path().join("nope"), now()).await;
|
|
assert_eq!(n, 0);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn paired_files_are_preserved() {
|
|
let tmp = TempDir::new().unwrap();
|
|
let stem = "uuid_2026-04-18T08-00-00Z";
|
|
touch_with_mtime(&tmp.path().join(format!("{stem}.m4a")), now()).await;
|
|
touch_with_mtime(&tmp.path().join(format!("{stem}.meta.json")), now()).await;
|
|
// Cutoff in the future → any orphan would be deleted — but
|
|
// these are paired, must survive.
|
|
let cutoff = now() + Duration::from_secs(3600);
|
|
let n = cleanup_orphan_audio(tmp.path(), cutoff).await;
|
|
assert_eq!(n, 0);
|
|
assert!(tmp.path().join(format!("{stem}.m4a")).exists());
|
|
assert!(tmp.path().join(format!("{stem}.meta.json")).exists());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn young_orphan_m4a_is_preserved() {
|
|
let tmp = TempDir::new().unwrap();
|
|
let m4a = tmp.path().join("uuid_young.m4a");
|
|
touch_with_mtime(&m4a, now()).await;
|
|
// cutoff = 1h ago → file (mtime=now) is younger → keep
|
|
let cutoff = now() - Duration::from_secs(3600);
|
|
assert_eq!(cleanup_orphan_audio(tmp.path(), cutoff).await, 0);
|
|
assert!(m4a.exists());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn old_orphan_m4a_is_deleted() {
|
|
let tmp = TempDir::new().unwrap();
|
|
let m4a = tmp.path().join("uuid_old.m4a");
|
|
touch_with_mtime(&m4a, now() - Duration::from_secs(48 * 3600)).await;
|
|
let cutoff = now() - Duration::from_secs(24 * 3600);
|
|
let n = cleanup_orphan_audio(tmp.path(), cutoff).await;
|
|
assert_eq!(n, 1);
|
|
assert!(!m4a.exists(), "old orphan m4a must be gone");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn old_orphan_sidecar_is_deleted() {
|
|
let tmp = TempDir::new().unwrap();
|
|
let sc = tmp.path().join("uuid_orphan.meta.json");
|
|
touch_with_mtime(&sc, now() - Duration::from_secs(48 * 3600)).await;
|
|
let cutoff = now() - Duration::from_secs(24 * 3600);
|
|
let n = cleanup_orphan_audio(tmp.path(), cutoff).await;
|
|
assert_eq!(n, 1);
|
|
assert!(!sc.exists());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn tmp_leftovers_always_removed_regardless_of_age() {
|
|
let tmp = TempDir::new().unwrap();
|
|
let young_tmp = tmp.path().join("fresh.meta.json.tmp");
|
|
let m4a_tmp = tmp.path().join("interrupted.m4a.tmp");
|
|
touch_with_mtime(&young_tmp, now()).await;
|
|
touch_with_mtime(&m4a_tmp, now()).await;
|
|
// Cutoff in the far past → age check would preserve everything.
|
|
// But tmp leftovers bypass age check.
|
|
let n = cleanup_orphan_audio(tmp.path(), now() - Duration::from_secs(10 * 86400)).await;
|
|
assert_eq!(n, 2);
|
|
assert!(!young_tmp.exists());
|
|
assert!(!m4a_tmp.exists());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn mix_of_pairs_orphans_and_tmp() {
|
|
let tmp = TempDir::new().unwrap();
|
|
let keep_stem = "uuid_keep_2026-04-18";
|
|
let orphan_stem = "uuid_orphan_old";
|
|
|
|
// Paired (keep)
|
|
touch_with_mtime(&tmp.path().join(format!("{keep_stem}.m4a")), now()).await;
|
|
touch_with_mtime(&tmp.path().join(format!("{keep_stem}.meta.json")), now()).await;
|
|
// Old orphan m4a (delete)
|
|
touch_with_mtime(
|
|
&tmp.path().join(format!("{orphan_stem}.m4a")),
|
|
now() - Duration::from_secs(48 * 3600),
|
|
)
|
|
.await;
|
|
// tmp leftover (delete)
|
|
touch_with_mtime(&tmp.path().join("crash.m4a.tmp"), now()).await;
|
|
|
|
let cutoff = now() - Duration::from_secs(24 * 3600);
|
|
let n = cleanup_orphan_audio(tmp.path(), cutoff).await;
|
|
assert_eq!(n, 2, "orphan + tmp must be removed, pair must stay");
|
|
assert!(tmp.path().join(format!("{keep_stem}.m4a")).exists());
|
|
assert!(tmp.path().join(format!("{keep_stem}.meta.json")).exists());
|
|
assert!(!tmp.path().join(format!("{orphan_stem}.m4a")).exists());
|
|
assert!(!tmp.path().join("crash.m4a.tmp").exists());
|
|
}
|
|
}
|