Refactor ETag generation for oneliners API
Remove the per-user oneliner watermark, which was previously used to generate ETags for the `/api/oneliners` endpoint. The watermark was intended to track the last successful oneliner write for each user. The ETag generation has been refactored to use a deterministic FNV-1a hash. This hash is computed over a sorted list of tuples containing `(case_id, last_recording_at_ns, oneliner_mtime_ns)` for all visible cases. This approach ensures that the ETag changes whenever any listen-relevant file system modification occurs, such as a new recording, oneliner regeneration, soft-delete, undo, or upload-auto-reopen. This new method is more robust and avoids the complexity of managing per-user state. The `OnelinerWatermark` type alias and its associated logic have been removed from `AppState` and the relevant modules (`transcribe::worker`, `transcribe::recovery`, `routes::oneliners`, `main`). New integration tests have been added to verify that various delete operations (single delete, undo delete, bulk delete) correctly trigger an ETag update, ensuring cache invalidation.
This commit is contained in:
@@ -5,16 +5,22 @@
|
||||
//! text (or null if not yet generated), and UTC timestamps for when the
|
||||
//! case started and when its oneliner was last written.
|
||||
//!
|
||||
//! Conditional-GET: the response carries a weak ETag `W/"<unix>-<hours>"`
|
||||
//! derived from the per-user watermark. Clients that send back a matching
|
||||
//! `If-None-Match` get `304 Not Modified` without touching the filesystem.
|
||||
//! Conditional-GET: the response carries a weak ETag
|
||||
//! `W/"<fingerprint_hex>-<hours>"`. The fingerprint is a deterministic
|
||||
//! FNV-1a hash over the sorted tuples
|
||||
//! `(case_id, last_recording_at_ns, oneliner_mtime_ns)` of every
|
||||
//! *visible* case. Any listen-relevant FS change — new recording,
|
||||
//! oneliner regeneration, soft-delete, undo, upload-auto-reopen —
|
||||
//! shifts at least one tuple and the hash moves. Server restart is a
|
||||
//! no-op: same FS, same hash.
|
||||
//!
|
||||
//! The `hours` component is included so a client swapping window sizes
|
||||
//! does not get a stale 304.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::SystemTime;
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::extract::Query;
|
||||
use axum::http::{header, HeaderMap, StatusCode};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::Json;
|
||||
@@ -26,7 +32,6 @@ use tracing::warn;
|
||||
|
||||
use crate::auth::AuthenticatedUser;
|
||||
use crate::error::AppError;
|
||||
use crate::OnelinerWatermark;
|
||||
|
||||
const DEFAULT_HOURS: u32 = 16;
|
||||
const MAX_HOURS: u32 = 168;
|
||||
@@ -38,7 +43,6 @@ pub struct OnelinersQuery {
|
||||
|
||||
pub async fn handle_oneliners(
|
||||
user: AuthenticatedUser,
|
||||
State(watermark): State<OnelinerWatermark>,
|
||||
Query(params): Query<OnelinersQuery>,
|
||||
headers: HeaderMap,
|
||||
) -> Result<Response, AppError> {
|
||||
@@ -46,18 +50,21 @@ pub async fn handle_oneliners(
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let since = now - Duration::hours(hours as i64);
|
||||
|
||||
let current_watermark = watermark.read().await.get(&user.slug).copied();
|
||||
let etag = build_etag(current_watermark, hours);
|
||||
// One scan pass yields both the case list and the fingerprint —
|
||||
// the fingerprint is the authoritative signal for "has the view
|
||||
// changed?". Body serialization is skipped if the client already
|
||||
// has this version (304).
|
||||
let scan = scan_cases(&user.data_dir, since).await;
|
||||
let etag = build_etag(scan.fingerprint, hours);
|
||||
|
||||
if client_has_matching_etag(&headers, &etag) {
|
||||
return Ok(not_modified(&etag));
|
||||
}
|
||||
|
||||
let oneliners = enumerate_recent_oneliners(&user.data_dir, since).await;
|
||||
let body = OnelinersResponse {
|
||||
as_of: now.format(&Rfc3339).unwrap_or_default(),
|
||||
window_hours: hours,
|
||||
oneliners,
|
||||
oneliners: scan.entries,
|
||||
};
|
||||
|
||||
let mut response = Json(body).into_response();
|
||||
@@ -67,9 +74,8 @@ pub async fn handle_oneliners(
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn build_etag(watermark: Option<OffsetDateTime>, hours: u32) -> String {
|
||||
let seconds = watermark.map(|t| t.unix_timestamp()).unwrap_or(0);
|
||||
format!("W/\"{seconds}-{hours}\"")
|
||||
fn build_etag(fingerprint: u64, hours: u32) -> String {
|
||||
format!("W/\"{fingerprint:016x}-{hours}\"")
|
||||
}
|
||||
|
||||
/// Case-sensitive match against the raw `If-None-Match` header value.
|
||||
@@ -91,19 +97,36 @@ fn not_modified(etag: &str) -> Response {
|
||||
response
|
||||
}
|
||||
|
||||
/// Output of [`scan_cases`]: the sorted entry list that becomes the
|
||||
/// response body, and a fingerprint derived from the same scan.
|
||||
struct ScanResult {
|
||||
entries: Vec<OnelinerEntry>,
|
||||
fingerprint: u64,
|
||||
}
|
||||
|
||||
/// Scan `<user_data_dir>/<case_id>/` for cases whose latest recording
|
||||
/// sits inside the `[since, now]` window. Returns entries sorted by
|
||||
/// `last_recording_at` descending (most recently active first); cases
|
||||
/// without recordings fall back to `created_at`. Soft-deleted cases and
|
||||
/// non-UUID directories are filtered out.
|
||||
async fn enumerate_recent_oneliners(
|
||||
user_data_dir: &Path,
|
||||
since: OffsetDateTime,
|
||||
) -> Vec<OnelinerEntry> {
|
||||
///
|
||||
/// Alongside the entries we compute a deterministic fingerprint
|
||||
/// (`fnv1a_64`) over the sorted set of
|
||||
/// `(case_id, last_recording_at_ns, oneliner_mtime_ns)` tuples — cheap
|
||||
/// to compute during the scan, and it changes exactly when the visible
|
||||
/// list changes. No RAM-local state, so server restart is a no-op.
|
||||
async fn scan_cases(user_data_dir: &Path, since: OffsetDateTime) -> ScanResult {
|
||||
let mut entries: Vec<OnelinerEntry> = Vec::new();
|
||||
let Ok(mut cases) = tokio::fs::read_dir(user_data_dir).await else {
|
||||
return entries;
|
||||
return ScanResult {
|
||||
entries,
|
||||
fingerprint: fnv1a_64(b""),
|
||||
};
|
||||
};
|
||||
// Parallel array keyed 1:1 with `entries` carrying the raw nanosecond
|
||||
// timestamps needed for the fingerprint. Kept separate so the
|
||||
// response DTO stays flat.
|
||||
let mut fp_tuples: Vec<(String, i128, i128)> = Vec::new();
|
||||
while let Ok(Some(case_entry)) = cases.next_entry().await {
|
||||
let case_dir = case_entry.path();
|
||||
if !case_dir.is_dir() {
|
||||
@@ -139,6 +162,10 @@ async fn enumerate_recent_oneliners(
|
||||
let last_recording_at_str = last_recording_at.format(&Rfc3339).ok();
|
||||
let updated_at_str = updated_at.and_then(|t| t.format(&Rfc3339).ok());
|
||||
|
||||
let last_rec_ns = last_recording_at.unix_timestamp_nanos();
|
||||
let oneliner_ns = updated_at.map(|t| t.unix_timestamp_nanos()).unwrap_or(0);
|
||||
fp_tuples.push((case_id.to_owned(), last_rec_ns, oneliner_ns));
|
||||
|
||||
entries.push(OnelinerEntry {
|
||||
case_id: case_id.to_owned(),
|
||||
oneliner: oneliner_text,
|
||||
@@ -156,9 +183,59 @@ async fn enumerate_recent_oneliners(
|
||||
let b_key = b.last_recording_at.as_deref().unwrap_or(&b.created_at);
|
||||
b_key.cmp(a_key)
|
||||
});
|
||||
entries
|
||||
|
||||
// Fingerprint: independent of entries' display order, so sort the
|
||||
// tuple list by case_id and hash.
|
||||
fp_tuples.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
let fingerprint = fingerprint_tuples(&fp_tuples);
|
||||
|
||||
ScanResult {
|
||||
entries,
|
||||
fingerprint,
|
||||
}
|
||||
}
|
||||
|
||||
/// FNV-1a 64-bit over the concatenation of the tuples. Stable across
|
||||
/// server restarts — no `RandomState`, no HashMap hashing. Ample
|
||||
/// resistance against accidental collisions for ETag purposes.
|
||||
fn fingerprint_tuples(tuples: &[(String, i128, i128)]) -> u64 {
|
||||
let mut h: u64 = FNV_OFFSET;
|
||||
for (case_id, last_rec_ns, oneliner_ns) in tuples {
|
||||
for b in case_id.as_bytes() {
|
||||
h ^= *b as u64;
|
||||
h = h.wrapping_mul(FNV_PRIME);
|
||||
}
|
||||
// Separator so `"ab" || "12"` ≠ `"a" || "b12"`.
|
||||
h ^= b'|' as u64;
|
||||
h = h.wrapping_mul(FNV_PRIME);
|
||||
for b in last_rec_ns.to_le_bytes() {
|
||||
h ^= b as u64;
|
||||
h = h.wrapping_mul(FNV_PRIME);
|
||||
}
|
||||
h ^= b'|' as u64;
|
||||
h = h.wrapping_mul(FNV_PRIME);
|
||||
for b in oneliner_ns.to_le_bytes() {
|
||||
h ^= b as u64;
|
||||
h = h.wrapping_mul(FNV_PRIME);
|
||||
}
|
||||
h ^= b'\n' as u64;
|
||||
h = h.wrapping_mul(FNV_PRIME);
|
||||
}
|
||||
h
|
||||
}
|
||||
|
||||
fn fnv1a_64(bytes: &[u8]) -> u64 {
|
||||
let mut h: u64 = FNV_OFFSET;
|
||||
for b in bytes {
|
||||
h ^= *b as u64;
|
||||
h = h.wrapping_mul(FNV_PRIME);
|
||||
}
|
||||
h
|
||||
}
|
||||
|
||||
const FNV_OFFSET: u64 = 0xcbf29ce4_84222325;
|
||||
const FNV_PRIME: u64 = 0x00000100_000001b3;
|
||||
|
||||
/// Return `(earliest, latest)` `.m4a` mtime pair in `case_dir`.
|
||||
/// `.m4a.failed` entries count — a failed recording still marks activity.
|
||||
/// `None` on empty directory or read error. A single recording yields
|
||||
|
||||
@@ -75,7 +75,9 @@ pub async fn handle_upload(
|
||||
// Auto-reopen soft-deleted cases: a new upload clears the `.deleted`
|
||||
// marker so the case reappears in /api/oneliners and the web UI.
|
||||
// Hard-deleted cases (directory already gone) are handled transparently
|
||||
// by `resolve_case_dir` above — nothing to clear.
|
||||
// by `resolve_case_dir` above — nothing to clear. No explicit ETag
|
||||
// bump is needed: the subsequent m4a write changes `last_recording_at`,
|
||||
// which flows into the `/api/oneliners` fingerprint automatically.
|
||||
if crate::paths::is_deleted(&case_dir).await {
|
||||
tokio::fs::remove_file(case_dir.join(crate::paths::DELETE_MARKER)).await?;
|
||||
info!(
|
||||
|
||||
Reference in New Issue
Block a user