Add recovery for missing oneliners
This commit introduces a recovery mechanism to regenerate `oneliner.txt` files for cases where transcription was successful but the oneliner generation failed due to a crash between writing the transcript and the oneliner. The `transcribe::recovery::regenerate_missing_oneliners` function is added and called at startup after `scan_and_enqueue`. This function identifies cases with non-empty transcripts but missing oneliners and triggers their regeneration. The `transcribe::worker::update_oneliner` function is refactored to always regenerate the oneliner from all available transcripts in a case, ensuring that later recordings can correct earlier ones. This replaces the previous `ensure_oneliner` logic which only generated the oneliner if it didn't exist. New helper functions `cases_needing_oneliner` and `all_transcripts_joined` are introduced to support these changes. Comprehensive unit tests are included for the new recovery and file processing logic.
This commit is contained in:
@@ -1,8 +1,9 @@
|
||||
use std::path::Path;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use tracing::{info, warn};
|
||||
|
||||
use super::{TranscribeJob, TranscribeSender};
|
||||
use crate::config::Config;
|
||||
|
||||
/// Walk `data_path/*/` and enqueue every recording pending transcription for
|
||||
/// every user. Intended for startup; the same primitive backs the per-user
|
||||
@@ -79,3 +80,137 @@ pub async fn enqueue_pending_for_user(
|
||||
}
|
||||
sent
|
||||
}
|
||||
|
||||
/// Walk `data_path/*/*` and return every case directory that has at least
|
||||
/// one non-empty `*.transcript.txt` but no `oneliner.txt`. Pure filesystem
|
||||
/// scan, no LLM calls — separated from the regeneration wrapper so it can
|
||||
/// be unit-tested without mocking Ollama.
|
||||
pub(crate) async fn cases_needing_oneliner(data_path: &Path) -> Vec<PathBuf> {
|
||||
let mut out: Vec<PathBuf> = Vec::new();
|
||||
let Ok(mut users) = tokio::fs::read_dir(data_path).await else {
|
||||
return out;
|
||||
};
|
||||
while let Ok(Some(user_entry)) = users.next_entry().await {
|
||||
let user_path = user_entry.path();
|
||||
if !user_path.is_dir() {
|
||||
continue;
|
||||
}
|
||||
let Ok(mut cases) = tokio::fs::read_dir(&user_path).await else {
|
||||
continue;
|
||||
};
|
||||
while let Ok(Some(case_entry)) = cases.next_entry().await {
|
||||
let case_dir = case_entry.path();
|
||||
if !case_dir.is_dir() || crate::paths::is_deleted(&case_dir).await {
|
||||
continue;
|
||||
}
|
||||
if case_dir.join("oneliner.txt").exists() {
|
||||
continue;
|
||||
}
|
||||
if has_non_empty_transcript(&case_dir).await {
|
||||
out.push(case_dir);
|
||||
}
|
||||
}
|
||||
}
|
||||
out.sort();
|
||||
out
|
||||
}
|
||||
|
||||
async fn has_non_empty_transcript(case_dir: &Path) -> bool {
|
||||
let Ok(mut files) = tokio::fs::read_dir(case_dir).await else {
|
||||
return false;
|
||||
};
|
||||
while let Ok(Some(f)) = files.next_entry().await {
|
||||
let path = f.path();
|
||||
let Some(name) = path.file_name().and_then(|s| s.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
if !name.ends_with(".transcript.txt") {
|
||||
continue;
|
||||
}
|
||||
if let Ok(text) = tokio::fs::read_to_string(&path).await
|
||||
&& !text.trim().is_empty()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Regenerate `oneliner.txt` for every case that has transcripts but no
|
||||
/// oneliner (crash between transcript-write and oneliner-write). Called at
|
||||
/// startup after `scan_and_enqueue`. Sequential on purpose — the oneliner
|
||||
/// model is light, but spamming it in parallel is not worth it.
|
||||
pub async fn regenerate_missing_oneliners(
|
||||
data_path: &Path,
|
||||
client: &reqwest::Client,
|
||||
config: &Config,
|
||||
) {
|
||||
let cases = cases_needing_oneliner(data_path).await;
|
||||
if cases.is_empty() {
|
||||
return;
|
||||
}
|
||||
info!(count = cases.len(), "Regenerating missing oneliners");
|
||||
for case_dir in cases {
|
||||
super::worker::update_oneliner(&case_dir, client, config).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn cases_needing_oneliner_finds_crash_survivor() {
|
||||
let data = tempdir().unwrap();
|
||||
let case = data.path().join("user").join("case1");
|
||||
tokio::fs::create_dir_all(&case).await.unwrap();
|
||||
tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), "content")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let got = cases_needing_oneliner(data.path()).await;
|
||||
assert_eq!(got, vec![case]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cases_needing_oneliner_skips_cases_with_oneliner() {
|
||||
let data = tempdir().unwrap();
|
||||
let case = data.path().join("user").join("case1");
|
||||
tokio::fs::create_dir_all(&case).await.unwrap();
|
||||
tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), "content")
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::fs::write(case.join("oneliner.txt"), "Existing")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(cases_needing_oneliner(data.path()).await.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cases_needing_oneliner_skips_cases_with_only_empty_transcripts() {
|
||||
let data = tempdir().unwrap();
|
||||
let case = data.path().join("user").join("case1");
|
||||
tokio::fs::create_dir_all(&case).await.unwrap();
|
||||
tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), " ")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(cases_needing_oneliner(data.path()).await.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cases_needing_oneliner_skips_deleted_cases() {
|
||||
let data = tempdir().unwrap();
|
||||
let case = data.path().join("user").join("case1");
|
||||
tokio::fs::create_dir_all(&case).await.unwrap();
|
||||
tokio::fs::write(case.join("2026-04-16T10-00-00Z.transcript.txt"), "content")
|
||||
.await
|
||||
.unwrap();
|
||||
// Empty JSON object is a valid delete marker (see paths::is_deleted).
|
||||
tokio::fs::write(case.join(".deleted"), "{}").await.unwrap();
|
||||
|
||||
assert!(cases_needing_oneliner(data.path()).await.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user