Refactor analyze recovery and worker busy status
The analyze recovery scan has been refactored to iterate through user directories and enqueue pending analysis jobs more efficiently. The `WorkerBusy` status has been introduced as an `Arc<AtomicBool>` to track whether the analyze worker is currently processing a job. This `WorkerBusy` flag is used in the `analyze::worker::run` function and managed by a `BusyGuard` RAII struct, ensuring the flag is correctly set and unset even in case of panics. The `handle_my_cases` and `handle_case_detail` handlers in `user_web.rs` now utilize the `WorkerBusy` flag to accurately display the "analyzing" status and to trigger a self-heal of orphaned analysis inputs when the worker is idle. Additionally, the `WorkerBusy` type is now exported from `server/src/lib.rs` to be accessible by other modules. The test cases have been updated to include the `WorkerBusy` parameter when spawning the analyze worker.
This commit is contained in:
@@ -1,78 +1,65 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::path::Path;
|
||||
|
||||
use tracing::{info, warn};
|
||||
|
||||
use super::{AnalyzeJob, AnalyzeSender};
|
||||
|
||||
/// Walk `data_path/*/*/analysis_input_v*.json` and enqueue every input
|
||||
/// that does not yet have a matching `document_v*.md` sibling.
|
||||
///
|
||||
/// Intended to run once at startup. Send is synchronous on the unbounded
|
||||
/// channel — only fails if the worker is gone, which is a programming error.
|
||||
/// Walk `data_path/*/` and enqueue every pending analysis for every user.
|
||||
/// Intended to run once at startup; the same primitive backs the per-user
|
||||
/// self-heal triggered by web handlers.
|
||||
pub async fn scan_and_enqueue(data_path: &Path, tx: &AnalyzeSender) {
|
||||
let pending = collect_pending(data_path).await;
|
||||
let total = pending.len();
|
||||
info!(count = total, "Analyze recovery scan found pending inputs");
|
||||
|
||||
for (case_dir, version) in pending {
|
||||
if tx.send(AnalyzeJob { case_dir: case_dir.clone(), version }).is_err() {
|
||||
warn!(case = %case_dir.display(), "Recovery send failed (worker gone)");
|
||||
break;
|
||||
}
|
||||
let Ok(mut users) = tokio::fs::read_dir(data_path).await else {
|
||||
return;
|
||||
};
|
||||
let mut total = 0;
|
||||
while let Ok(Some(user_entry)) = users.next_entry().await {
|
||||
total += enqueue_pending_for_user(&user_entry.path(), tx).await;
|
||||
}
|
||||
info!(count = total, "Analyze recovery scan finished");
|
||||
}
|
||||
|
||||
/// Returns `(case_dir, version)` tuples for every `analysis_input_v{N}.json`
|
||||
/// whose `document_v{N}.md` is missing.
|
||||
async fn collect_pending(data_path: &Path) -> Vec<(PathBuf, u32)> {
|
||||
let mut out: Vec<(PathBuf, u32)> = Vec::new();
|
||||
|
||||
let mut users = match tokio::fs::read_dir(data_path).await {
|
||||
Ok(r) => r,
|
||||
Err(_) => return out,
|
||||
/// Scan a single `<user_root>/<case>/analysis_input_v*.json` set and enqueue
|
||||
/// every input whose `document_v{N}.md` is missing. Returns the number sent.
|
||||
/// Channel send is synchronous on the unbounded channel — only fails if the
|
||||
/// worker is gone, which is a programming error.
|
||||
pub async fn enqueue_pending_for_user(user_root: &Path, tx: &AnalyzeSender) -> usize {
|
||||
let Ok(mut cases) = tokio::fs::read_dir(user_root).await else {
|
||||
return 0;
|
||||
};
|
||||
|
||||
while let Ok(Some(user_entry)) = users.next_entry().await {
|
||||
let mut cases = match tokio::fs::read_dir(user_entry.path()).await {
|
||||
Ok(r) => r,
|
||||
Err(_) => continue,
|
||||
let mut sent = 0;
|
||||
while let Ok(Some(case_entry)) = cases.next_entry().await {
|
||||
let case_dir = case_entry.path();
|
||||
if !case_dir.is_dir() || crate::paths::is_deleted(&case_dir).await {
|
||||
continue;
|
||||
}
|
||||
let Ok(mut files) = tokio::fs::read_dir(&case_dir).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
while let Ok(Some(case_entry)) = cases.next_entry().await {
|
||||
let case_dir = case_entry.path();
|
||||
if !case_dir.is_dir() {
|
||||
while let Ok(Some(file)) = files.next_entry().await {
|
||||
let path = file.path();
|
||||
let Some(name) = path.file_name().and_then(|s| s.to_str()) else {
|
||||
continue;
|
||||
}
|
||||
if crate::paths::is_deleted(&case_dir).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut files = match tokio::fs::read_dir(&case_dir).await {
|
||||
Ok(r) => r,
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
||||
while let Ok(Some(file)) = files.next_entry().await {
|
||||
let path = file.path();
|
||||
let Some(name) = path.file_name().and_then(|s| s.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
let Some(version) = parse_input_version(name) else {
|
||||
continue;
|
||||
};
|
||||
let document = case_dir.join(format!("document_v{version}.md"));
|
||||
if !document.exists() {
|
||||
out.push((case_dir.clone(), version));
|
||||
}
|
||||
let Some(version) = parse_input_version(name) else {
|
||||
continue;
|
||||
};
|
||||
if case_dir.join(format!("document_v{version}.md")).exists() {
|
||||
continue;
|
||||
}
|
||||
if tx
|
||||
.send(AnalyzeJob {
|
||||
case_dir: case_dir.clone(),
|
||||
version,
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
warn!(case = %case_dir.display(), "recovery send failed (worker gone)");
|
||||
return sent;
|
||||
}
|
||||
sent += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Oldest case directories first — not strictly required for correctness,
|
||||
// but gives deterministic recovery order. Cases are UUIDs so sorting is
|
||||
// purely lexicographic, which is fine.
|
||||
out.sort();
|
||||
out
|
||||
sent
|
||||
}
|
||||
|
||||
/// Parse `analysis_input_v{N}.json` and return `N` (as u32) if the pattern
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -7,15 +8,40 @@ use tracing::{error, info, warn};
|
||||
|
||||
use super::{llm, prompt, AnalysisInput, AnalyzeReceiver};
|
||||
use crate::config::Config;
|
||||
use crate::WorkerBusy;
|
||||
|
||||
/// RAII guard that flips `WorkerBusy` to `true` on construction and back to
|
||||
/// `false` on drop. Panic-safe: even if `process()` unwinds, the flag is
|
||||
/// cleared.
|
||||
struct BusyGuard(WorkerBusy);
|
||||
|
||||
impl BusyGuard {
|
||||
fn new(busy: WorkerBusy) -> Self {
|
||||
busy.store(true, Ordering::Release);
|
||||
Self(busy)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BusyGuard {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(false, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
/// Consume analyze jobs sequentially. The external LLM tolerates parallelism,
|
||||
/// but sequential processing keeps the architecture simple and shields the
|
||||
/// provider from bursts. Trivial to lift later via `tokio::spawn(process(..))`.
|
||||
pub async fn run(mut rx: AnalyzeReceiver, config: Arc<Config>, client: reqwest::Client) {
|
||||
pub async fn run(
|
||||
mut rx: AnalyzeReceiver,
|
||||
config: Arc<Config>,
|
||||
client: reqwest::Client,
|
||||
worker_busy: WorkerBusy,
|
||||
) {
|
||||
info!("Analyze worker started");
|
||||
let timeout = Duration::from_secs(config.llm_timeout_seconds);
|
||||
|
||||
while let Some(job) = rx.recv().await {
|
||||
let _guard = BusyGuard::new(worker_busy.clone());
|
||||
process(&job.case_dir, job.version, &config, &client, timeout).await;
|
||||
}
|
||||
|
||||
@@ -100,6 +126,13 @@ async fn process(
|
||||
return;
|
||||
}
|
||||
|
||||
// Job done — drop the queue marker so the live "analyzing" flag
|
||||
// becomes accurate immediately and recovery on next boot does not
|
||||
// re-enqueue this version.
|
||||
if let Err(e) = tokio::fs::remove_file(&input_path).await {
|
||||
warn!(path = %input_path.display(), error = %e, "removing analysis input failed");
|
||||
}
|
||||
|
||||
info!(
|
||||
case = %case_dir.display(),
|
||||
version,
|
||||
|
||||
Reference in New Issue
Block a user