feat: multi-backend LLM layer with per-request choice on case page
Replace the single [llm] block in settings.toml with a curated catalog of LLM "backends" (provider + model + sampling params + system prompt) in server/src/analyze/backend.rs. Two backends ship initially: gpt_oss_120b (default, reasoning_effort=medium) and llama_3_1_405b (uses response_format: json_schema to sidestep the <|eot_id|> leak). The case page now renders one submit button per available backend; the clicked button's name=backend value rides through AnalysisInput.backend_id to the worker, which looks it up via find_backend() with default fallback. A submitted unknown backend id is rejected as 400. .analysis_failed.json records the failed backend and the failure banner surfaces its label. The API key now comes from IONOS_API_KEY (env), not settings.toml; the [llm] section is read into a discarded field so old configs still load. Backends without a satisfied requires_api_key are filtered from the UI (any_backend_available()). Three end-to-end tests that mocked the LLM endpoint via settings.llm.url are #[ignore]'d with a clear note — they need per-test backend injection (Arc<Vec<LlmBackend>> through the worker) before they can be re-enabled.
This commit is contained in:
@@ -357,6 +357,10 @@ struct CasePageTemplate {
|
||||
/// `is_admin` is false (template skips the block in that case anyway).
|
||||
/// Pre-sanitised against `</script>` breakout via `<\/` substitution.
|
||||
debug_copy_json: String,
|
||||
/// Available LLM backends (filtered to those whose auth is satisfied).
|
||||
/// One submit button per entry is rendered next to the analyze form.
|
||||
/// The first entry is the default backend.
|
||||
backends: Vec<&'static crate::analyze::backend::LlmBackend>,
|
||||
}
|
||||
|
||||
#[derive(Template)]
|
||||
@@ -388,6 +392,10 @@ struct FailureBanner {
|
||||
/// into the doctor's local timezone (same `time_format.js` that
|
||||
/// formats `recorded_at_iso`).
|
||||
failed_at: String,
|
||||
/// UI label of the backend that produced the failure (resolved from
|
||||
/// the marker's `backend_id`). Empty if the marker predates the
|
||||
/// per-backend tracking or the id is no longer in the catalog.
|
||||
backend_label: String,
|
||||
}
|
||||
|
||||
/// Flags derived from filesystem state + config.
|
||||
@@ -432,6 +440,12 @@ async fn compute_flags(
|
||||
.map(|m| FailureBanner {
|
||||
reason: m.reason,
|
||||
failed_at: m.failed_at,
|
||||
backend_label: m
|
||||
.backend_id
|
||||
.as_deref()
|
||||
.and_then(crate::analyze::backend::find_backend)
|
||||
.map(|b| b.label.clone())
|
||||
.unwrap_or_default(),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
@@ -556,8 +570,7 @@ pub async fn handle_my_cases(
|
||||
// render. The common case is "nothing to do" and costs a handful of
|
||||
// stat-calls per case; actual enqueues happen only when pre-conditions
|
||||
// flip (new transcripts in, stale document, ...).
|
||||
auto_trigger::try_enqueue_all_for_user(&user_root, &pipeline.analyze_tx, &settings, &events_tx)
|
||||
.await;
|
||||
auto_trigger::try_enqueue_all_for_user(&user_root, &pipeline.analyze_tx, &events_tx).await;
|
||||
|
||||
// Lazy retention sweep: at most one disk scan per /web/cases visit,
|
||||
// reusing the user's retention policy from users.toml. Runs before
|
||||
@@ -579,7 +592,7 @@ pub async fn handle_my_cases(
|
||||
busy,
|
||||
show_closed,
|
||||
retention.auto_delete_days,
|
||||
settings.llm_configured(),
|
||||
crate::analyze::backend::any_backend_available(),
|
||||
)
|
||||
.await;
|
||||
let total = cases.len();
|
||||
@@ -669,7 +682,7 @@ pub async fn handle_case_page(
|
||||
// Auto-analysis trigger for deep-link navigation: same evaluation as
|
||||
// the list view. Document render below still wins the race if the
|
||||
// worker happens to finish synchronously.
|
||||
auto_trigger::try_enqueue(&case_dir, &pipeline.analyze_tx, &settings, &events_tx).await;
|
||||
auto_trigger::try_enqueue(&case_dir, &pipeline.analyze_tx, &events_tx).await;
|
||||
let case_id_str = case_id.to_string();
|
||||
info!(slug = %user.slug, case_id = %case_id, "case page viewed");
|
||||
|
||||
@@ -687,7 +700,13 @@ pub async fn handle_case_page(
|
||||
let (oneliner, oneliner_is_manual) = compute_oneliner_display(&case_dir, &recordings).await;
|
||||
|
||||
let a_busy = pipeline.analyze_busy.0.load(Ordering::Acquire);
|
||||
let flags = compute_flags(&case_dir, &recordings, settings.llm_configured(), a_busy).await;
|
||||
let flags = compute_flags(
|
||||
&case_dir,
|
||||
&recordings,
|
||||
crate::analyze::backend::any_backend_available(),
|
||||
a_busy,
|
||||
)
|
||||
.await;
|
||||
|
||||
let recordings_count = recordings.len();
|
||||
let transcribed_count = recordings
|
||||
@@ -747,6 +766,7 @@ pub async fn handle_case_page(
|
||||
is_closed,
|
||||
csrf_token: user.csrf_token,
|
||||
debug_copy_json,
|
||||
backends: crate::analyze::backend::available_backends(),
|
||||
}
|
||||
.render()
|
||||
.map(Html)
|
||||
|
||||
Reference in New Issue
Block a user