feat: Add API endpoint for listing oneliners
This commit introduces a new API endpoint `/api/oneliners` that allows authenticated users to retrieve a list of their recent oneliners. The endpoint supports conditional GET requests using the `ETag` header for efficient caching. It also allows specifying a time window for the oneliners via the `hours` query parameter, with sensible defaults and clamping. The implementation includes: - A new route handler `handle_oneliners`. - A new type `OnelinerWatermark` for tracking user-specific timestamps. - Logic for scanning user data directories, filtering by time, and handling deleted cases. - Test cases to cover various scenarios, including caching, time windowing, and edge cases.
This commit is contained in:
@@ -9,17 +9,27 @@ pub mod routes;
|
||||
pub mod transcribe;
|
||||
pub mod web_session;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::extract::FromRef;
|
||||
use axum::Router;
|
||||
use time::OffsetDateTime;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use analyze::AnalyzeSender;
|
||||
use config::Config;
|
||||
use transcribe::TranscribeSender;
|
||||
use web_session::SessionStore;
|
||||
|
||||
/// Per-user UTC timestamp of the last successful oneliner write.
|
||||
/// Read on every `/api/oneliners` request to build the ETag; written
|
||||
/// from the transcribe worker and the boot recovery scan. Missing entry
|
||||
/// → the user has no known oneliner activity yet; the endpoint returns
|
||||
/// a full response and sets an ETag based on a `0` seed.
|
||||
pub type OnelinerWatermark = Arc<RwLock<HashMap<String, OffsetDateTime>>>;
|
||||
|
||||
/// Live "is this worker currently processing a job?" flag.
|
||||
/// Set true before each job, false after. UI uses it to distinguish a real
|
||||
/// in-flight job from a stale on-disk marker (orphan input, missing
|
||||
@@ -60,6 +70,7 @@ pub struct AppState {
|
||||
pub session_store: SessionStore,
|
||||
pub analyze_busy: AnalyzeBusy,
|
||||
pub transcribe_busy: TranscribeBusy,
|
||||
pub oneliner_watermark: OnelinerWatermark,
|
||||
}
|
||||
|
||||
impl FromRef<AppState> for Arc<Config> {
|
||||
@@ -98,6 +109,12 @@ impl FromRef<AppState> for TranscribeBusy {
|
||||
}
|
||||
}
|
||||
|
||||
impl FromRef<AppState> for OnelinerWatermark {
|
||||
fn from_ref(state: &AppState) -> Self {
|
||||
state.oneliner_watermark.clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// Test/simple entrypoint: jobs pushed into either channel are dropped
|
||||
/// because the receivers are not retained. Use [`create_router_with_state`]
|
||||
/// from `main.rs` where real workers own the receivers.
|
||||
@@ -111,6 +128,7 @@ pub fn create_router(config: Arc<Config>) -> Router {
|
||||
session_store: web_session::new_store(),
|
||||
analyze_busy: AnalyzeBusy(Arc::new(AtomicBool::new(false))),
|
||||
transcribe_busy: TranscribeBusy(Arc::new(AtomicBool::new(false))),
|
||||
oneliner_watermark: Arc::new(RwLock::new(HashMap::new())),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+15
-2
@@ -12,7 +12,7 @@ use doctate_server::analyze;
|
||||
use doctate_server::config::Config;
|
||||
use doctate_server::gazetteer::{Gazetteer, SpellbookDict};
|
||||
use doctate_server::transcribe;
|
||||
use doctate_server::AppState;
|
||||
use doctate_server::{AppState, OnelinerWatermark};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
@@ -110,6 +110,13 @@ async fn main() {
|
||||
}
|
||||
});
|
||||
|
||||
// Oneliner watermark: per-user "last successful oneliner write (UTC)".
|
||||
// Feeds the ETag on `/api/oneliners`. Shared across the transcribe
|
||||
// worker (writer) and the oneliners route handler (reader).
|
||||
let oneliner_watermark: OnelinerWatermark = Arc::new(tokio::sync::RwLock::new(
|
||||
std::collections::HashMap::new(),
|
||||
));
|
||||
|
||||
// Transcription pipeline: channel + worker + recovery scan.
|
||||
let (transcribe_tx, transcribe_rx) = transcribe::channel();
|
||||
let transcribe_busy: doctate_server::WorkerBusy =
|
||||
@@ -120,6 +127,7 @@ async fn main() {
|
||||
http_client.clone(),
|
||||
transcribe_busy.clone(),
|
||||
vocab.clone(),
|
||||
oneliner_watermark.clone(),
|
||||
));
|
||||
{
|
||||
let tx = transcribe_tx.clone();
|
||||
@@ -127,11 +135,15 @@ async fn main() {
|
||||
let client = http_client.clone();
|
||||
let cfg = config.clone();
|
||||
let vocab = vocab.clone();
|
||||
let watermark = oneliner_watermark.clone();
|
||||
tokio::spawn(async move {
|
||||
transcribe::recovery::scan_and_enqueue(&data_path, &tx).await;
|
||||
// Then catch up oneliners for cases that crashed between
|
||||
// transcript-write and oneliner-write in a previous run.
|
||||
transcribe::recovery::regenerate_missing_oneliners(&data_path, &client, &cfg, &vocab).await;
|
||||
transcribe::recovery::regenerate_missing_oneliners(
|
||||
&data_path, &client, &cfg, &vocab, &watermark,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -161,6 +173,7 @@ async fn main() {
|
||||
session_store: doctate_server::web_session::new_store(),
|
||||
analyze_busy: doctate_server::AnalyzeBusy(analyze_busy),
|
||||
transcribe_busy: doctate_server::TranscribeBusy(transcribe_busy),
|
||||
oneliner_watermark,
|
||||
};
|
||||
|
||||
let addr = format!("0.0.0.0:{}", config.server_port);
|
||||
|
||||
@@ -3,6 +3,7 @@ mod case_actions;
|
||||
mod debug;
|
||||
mod health;
|
||||
mod login;
|
||||
mod oneliners;
|
||||
mod upload;
|
||||
pub(crate) mod user_web;
|
||||
pub(crate) mod web;
|
||||
@@ -17,6 +18,7 @@ pub fn api_router() -> Router<AppState> {
|
||||
.route("/api/health", get(health::handle_health))
|
||||
.route("/api/debug/whoami", get(debug::handle_whoami))
|
||||
.route("/api/upload", post(upload::handle_upload))
|
||||
.route("/api/oneliners", get(oneliners::handle_oneliners))
|
||||
.route("/web/login", get(login::handle_login_page).post(login::handle_login_submit))
|
||||
.route("/web/logout", post(login::handle_logout))
|
||||
.route("/web/cases", get(user_web::handle_my_cases))
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
//! GET /api/oneliners — list oneliners for the authenticated user.
|
||||
//!
|
||||
//! Time window is caller-controlled via `?hours=N` (default 16, clamped
|
||||
//! to [1, MAX_HOURS]). Each entry carries the case-id, the oneliner
|
||||
//! text (or null if not yet generated), and UTC timestamps for when the
|
||||
//! case started and when its oneliner was last written.
|
||||
//!
|
||||
//! Conditional-GET: the response carries a weak ETag `W/"<unix>-<hours>"`
|
||||
//! derived from the per-user watermark. Clients that send back a matching
|
||||
//! `If-None-Match` get `304 Not Modified` without touching the filesystem.
|
||||
//! The `hours` component is included so a client swapping window sizes
|
||||
//! does not get a stale 304.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::SystemTime;
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::http::{header, HeaderMap, StatusCode};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::Json;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
use time::{Duration, OffsetDateTime};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::auth::AuthenticatedUser;
|
||||
use crate::error::AppError;
|
||||
use crate::OnelinerWatermark;
|
||||
|
||||
const DEFAULT_HOURS: u32 = 16;
|
||||
const MAX_HOURS: u32 = 168;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct OnelinersQuery {
|
||||
hours: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct OnelinersResponse {
|
||||
as_of: String,
|
||||
window_hours: u32,
|
||||
oneliners: Vec<OnelinerEntry>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct OnelinerEntry {
|
||||
case_id: String,
|
||||
oneliner: Option<String>,
|
||||
created_at: String,
|
||||
updated_at: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn handle_oneliners(
|
||||
user: AuthenticatedUser,
|
||||
State(watermark): State<OnelinerWatermark>,
|
||||
Query(params): Query<OnelinersQuery>,
|
||||
headers: HeaderMap,
|
||||
) -> Result<Response, AppError> {
|
||||
let hours = params.hours.unwrap_or(DEFAULT_HOURS).clamp(1, MAX_HOURS);
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let since = now - Duration::hours(hours as i64);
|
||||
|
||||
let current_watermark = watermark.read().await.get(&user.slug).copied();
|
||||
let etag = build_etag(current_watermark, hours);
|
||||
|
||||
if client_has_matching_etag(&headers, &etag) {
|
||||
return Ok(not_modified(&etag));
|
||||
}
|
||||
|
||||
let oneliners = enumerate_recent_oneliners(&user.data_dir, since).await;
|
||||
let body = OnelinersResponse {
|
||||
as_of: now.format(&Rfc3339).unwrap_or_default(),
|
||||
window_hours: hours,
|
||||
oneliners,
|
||||
};
|
||||
|
||||
let mut response = Json(body).into_response();
|
||||
if let Ok(value) = etag.parse() {
|
||||
response.headers_mut().insert(header::ETAG, value);
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn build_etag(watermark: Option<OffsetDateTime>, hours: u32) -> String {
|
||||
let seconds = watermark.map(|t| t.unix_timestamp()).unwrap_or(0);
|
||||
format!("W/\"{seconds}-{hours}\"")
|
||||
}
|
||||
|
||||
/// Case-sensitive match against the raw `If-None-Match` header value.
|
||||
/// No wildcard or comma-split logic — the client is our own code sending
|
||||
/// back the exact ETag we handed it.
|
||||
fn client_has_matching_etag(headers: &HeaderMap, etag: &str) -> bool {
|
||||
headers
|
||||
.get(header::IF_NONE_MATCH)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|sent| sent == etag)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn not_modified(etag: &str) -> Response {
|
||||
let mut response = StatusCode::NOT_MODIFIED.into_response();
|
||||
if let Ok(value) = etag.parse() {
|
||||
response.headers_mut().insert(header::ETAG, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
/// Scan `<user_data_dir>/<case_id>/` for cases whose earliest recording
|
||||
/// sits inside the `[since, now]` window. Returns entries sorted by
|
||||
/// `created_at` descending (newest first). Soft-deleted cases and
|
||||
/// non-UUID directories are filtered out.
|
||||
async fn enumerate_recent_oneliners(
|
||||
user_data_dir: &Path,
|
||||
since: OffsetDateTime,
|
||||
) -> Vec<OnelinerEntry> {
|
||||
let mut entries: Vec<OnelinerEntry> = Vec::new();
|
||||
let Ok(mut cases) = tokio::fs::read_dir(user_data_dir).await else {
|
||||
return entries;
|
||||
};
|
||||
while let Ok(Some(case_entry)) = cases.next_entry().await {
|
||||
let case_dir = case_entry.path();
|
||||
if !case_dir.is_dir() {
|
||||
continue;
|
||||
}
|
||||
let Some(case_id) = case_dir.file_name().and_then(|s| s.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
if uuid::Uuid::parse_str(case_id).is_err() {
|
||||
continue;
|
||||
}
|
||||
if crate::paths::is_deleted(&case_dir).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(created_mtime) = earliest_m4a_mtime(&case_dir).await else {
|
||||
continue;
|
||||
};
|
||||
let created_at = OffsetDateTime::from(created_mtime);
|
||||
if created_at < since {
|
||||
continue;
|
||||
}
|
||||
|
||||
let (oneliner_text, updated_at) = read_oneliner(&case_dir).await;
|
||||
let Ok(created_at_str) = created_at.format(&Rfc3339) else {
|
||||
warn!(case = %case_dir.display(), "format created_at failed");
|
||||
continue;
|
||||
};
|
||||
let updated_at_str = updated_at.and_then(|t| t.format(&Rfc3339).ok());
|
||||
|
||||
entries.push(OnelinerEntry {
|
||||
case_id: case_id.to_owned(),
|
||||
oneliner: oneliner_text,
|
||||
created_at: created_at_str,
|
||||
updated_at: updated_at_str,
|
||||
});
|
||||
}
|
||||
entries.sort_by(|a, b| b.created_at.cmp(&a.created_at));
|
||||
entries
|
||||
}
|
||||
|
||||
/// Return the earliest `.m4a` mtime in `case_dir`. `.m4a.failed` counts —
|
||||
/// a failed recording still marks the birth of the case. Returns None on
|
||||
/// read errors or empty directories.
|
||||
async fn earliest_m4a_mtime(case_dir: &Path) -> Option<SystemTime> {
|
||||
let mut files = tokio::fs::read_dir(case_dir).await.ok()?;
|
||||
let mut earliest: Option<SystemTime> = None;
|
||||
while let Ok(Some(entry)) = files.next_entry().await {
|
||||
let path: PathBuf = entry.path();
|
||||
let Some(name) = path.file_name().and_then(|s| s.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
if !(name.ends_with(".m4a") || name.ends_with(".m4a.failed")) {
|
||||
continue;
|
||||
}
|
||||
let Ok(meta) = entry.metadata().await else {
|
||||
continue;
|
||||
};
|
||||
let Ok(mtime) = meta.modified() else {
|
||||
continue;
|
||||
};
|
||||
earliest = match earliest {
|
||||
None => Some(mtime),
|
||||
Some(e) if mtime < e => Some(mtime),
|
||||
other => other,
|
||||
};
|
||||
}
|
||||
earliest
|
||||
}
|
||||
|
||||
/// Read `oneliner.txt` (trimmed, non-empty) plus its mtime. Missing file
|
||||
/// yields `(None, None)`.
|
||||
async fn read_oneliner(case_dir: &Path) -> (Option<String>, Option<OffsetDateTime>) {
|
||||
let path = case_dir.join("oneliner.txt");
|
||||
let text = tokio::fs::read_to_string(&path)
|
||||
.await
|
||||
.ok()
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty());
|
||||
if text.is_none() {
|
||||
return (None, None);
|
||||
}
|
||||
let mtime = tokio::fs::metadata(&path)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|m| m.modified().ok())
|
||||
.map(OffsetDateTime::from);
|
||||
(text, mtime)
|
||||
}
|
||||
@@ -5,6 +5,7 @@ use tracing::{info, warn};
|
||||
use super::{TranscribeJob, TranscribeSender};
|
||||
use crate::config::Config;
|
||||
use crate::gazetteer::Gazetteer;
|
||||
use crate::OnelinerWatermark;
|
||||
|
||||
/// Walk `data_path/*/` and enqueue every recording pending transcription for
|
||||
/// every user. Intended for startup; the same primitive backs the per-user
|
||||
@@ -83,11 +84,13 @@ pub async fn enqueue_pending_for_user(
|
||||
}
|
||||
|
||||
/// Walk `data_path/*/*` and return every case directory that has at least
|
||||
/// one non-empty `*.transcript.txt` but no `oneliner.txt`. Pure filesystem
|
||||
/// scan, no LLM calls — separated from the regeneration wrapper so it can
|
||||
/// be unit-tested without mocking Ollama.
|
||||
pub(crate) async fn cases_needing_oneliner(data_path: &Path) -> Vec<PathBuf> {
|
||||
let mut out: Vec<PathBuf> = Vec::new();
|
||||
/// one non-empty `*.transcript.txt` but no `oneliner.txt`, paired with the
|
||||
/// user-slug (parent directory name). Pure filesystem scan, no LLM calls —
|
||||
/// separated from the regeneration wrapper so it can be unit-tested without
|
||||
/// mocking Ollama. The slug is carried through so the caller can bump the
|
||||
/// per-user watermark when regeneration succeeds.
|
||||
pub(crate) async fn cases_needing_oneliner(data_path: &Path) -> Vec<(PathBuf, String)> {
|
||||
let mut out: Vec<(PathBuf, String)> = Vec::new();
|
||||
let Ok(mut users) = tokio::fs::read_dir(data_path).await else {
|
||||
return out;
|
||||
};
|
||||
@@ -96,6 +99,9 @@ pub(crate) async fn cases_needing_oneliner(data_path: &Path) -> Vec<PathBuf> {
|
||||
if !user_path.is_dir() {
|
||||
continue;
|
||||
}
|
||||
let Some(slug) = user_entry.file_name().to_str().map(str::to_owned) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(mut cases) = tokio::fs::read_dir(&user_path).await else {
|
||||
continue;
|
||||
};
|
||||
@@ -108,11 +114,11 @@ pub(crate) async fn cases_needing_oneliner(data_path: &Path) -> Vec<PathBuf> {
|
||||
continue;
|
||||
}
|
||||
if has_non_empty_transcript(&case_dir).await {
|
||||
out.push(case_dir);
|
||||
out.push((case_dir, slug.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
out.sort();
|
||||
out.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
out
|
||||
}
|
||||
|
||||
@@ -146,14 +152,15 @@ pub async fn regenerate_missing_oneliners(
|
||||
client: &reqwest::Client,
|
||||
config: &Config,
|
||||
vocab: &Gazetteer,
|
||||
watermark: &OnelinerWatermark,
|
||||
) {
|
||||
let cases = cases_needing_oneliner(data_path).await;
|
||||
if cases.is_empty() {
|
||||
return;
|
||||
}
|
||||
info!(count = cases.len(), "Regenerating missing oneliners");
|
||||
for case_dir in cases {
|
||||
super::worker::update_oneliner(&case_dir, client, config, vocab).await;
|
||||
for (case_dir, slug) in cases {
|
||||
super::worker::update_oneliner(&case_dir, &slug, client, config, vocab, watermark).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,7 +179,7 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
let got = cases_needing_oneliner(data.path()).await;
|
||||
assert_eq!(got, vec![case]);
|
||||
assert_eq!(got, vec![(case, "user".to_owned())]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -2,12 +2,13 @@ use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use time::OffsetDateTime;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
use super::{ffmpeg, ollama, whisper, TranscribeReceiver};
|
||||
use crate::config::{Config, WhisperUserSettings};
|
||||
use crate::gazetteer::Gazetteer;
|
||||
use crate::{BusyGuard, WorkerBusy};
|
||||
use crate::{BusyGuard, OnelinerWatermark, WorkerBusy};
|
||||
|
||||
const ONELINER_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
@@ -19,6 +20,7 @@ pub async fn run(
|
||||
client: reqwest::Client,
|
||||
worker_busy: WorkerBusy,
|
||||
vocab: Arc<Gazetteer>,
|
||||
watermark: OnelinerWatermark,
|
||||
) {
|
||||
info!("Transcription worker started");
|
||||
let timeout = Duration::from_secs(config.whisper_timeout_seconds);
|
||||
@@ -88,7 +90,7 @@ pub async fn run(
|
||||
if let Some(case_dir) = audio_path.parent()
|
||||
&& !has_pending_recordings(case_dir).await
|
||||
{
|
||||
update_oneliner(case_dir, &client, &config, &vocab).await;
|
||||
update_oneliner(case_dir, &job.user_slug, &client, &config, &vocab, &watermark).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,9 +129,11 @@ async fn mark_failed(audio_path: &Path) {
|
||||
/// no-op — next transcript write retries.
|
||||
pub(crate) async fn update_oneliner(
|
||||
case_dir: &Path,
|
||||
user_slug: &str,
|
||||
client: &reqwest::Client,
|
||||
config: &Config,
|
||||
vocab: &Gazetteer,
|
||||
watermark: &OnelinerWatermark,
|
||||
) {
|
||||
let path = case_dir.join("oneliner.txt");
|
||||
|
||||
@@ -154,6 +158,12 @@ pub(crate) async fn update_oneliner(
|
||||
error!(path = %path.display(), error = %e, "writing oneliner failed");
|
||||
} else {
|
||||
info!(path = %path.display(), chars = line.chars().count(), "Oneliner updated");
|
||||
// Bump the per-user watermark so the next `/api/oneliners`
|
||||
// poll sees a fresh ETag and fetches the new content.
|
||||
watermark
|
||||
.write()
|
||||
.await
|
||||
.insert(user_slug.to_string(), OffsetDateTime::now_utc());
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
|
||||
Reference in New Issue
Block a user