feat: multi-backend LLM layer with per-request choice on case page

Replace the single [llm] block in settings.toml with a curated catalog
of LLM "backends" (provider + model + sampling params + system prompt)
in server/src/analyze/backend.rs. Two backends ship initially:
gpt_oss_120b (default, reasoning_effort=medium) and llama_3_1_405b
(uses response_format: json_schema to sidestep the <|eot_id|> leak).

The case page now renders one submit button per available backend; the
clicked button's name=backend value rides through AnalysisInput.backend_id
to the worker, which looks it up via find_backend() with default fallback.
A submitted unknown backend id is rejected as 400. .analysis_failed.json
records the failed backend and the failure banner surfaces its label.

The API key now comes from IONOS_API_KEY (env), not settings.toml; the
[llm] section is read into a discarded field so old configs still load.
Backends without a satisfied requires_api_key are filtered from the UI
(any_backend_available()). Three end-to-end tests that mocked the LLM
endpoint via settings.llm.url are #[ignore]'d with a clear note — they
need per-test backend injection (Arc<Vec<LlmBackend>> through the worker)
before they can be re-enabled.
This commit is contained in:
2026-05-03 14:57:21 +02:00
parent cbb072d0cc
commit 23ef84d9d8
18 changed files with 596 additions and 307 deletions
+25 -5
View File
@@ -357,6 +357,10 @@ struct CasePageTemplate {
/// `is_admin` is false (template skips the block in that case anyway).
/// Pre-sanitised against `</script>` breakout via `<\/` substitution.
debug_copy_json: String,
/// Available LLM backends (filtered to those whose auth is satisfied).
/// One submit button per entry is rendered next to the analyze form.
/// The first entry is the default backend.
backends: Vec<&'static crate::analyze::backend::LlmBackend>,
}
#[derive(Template)]
@@ -388,6 +392,10 @@ struct FailureBanner {
/// into the doctor's local timezone (same `time_format.js` that
/// formats `recorded_at_iso`).
failed_at: String,
/// UI label of the backend that produced the failure (resolved from
/// the marker's `backend_id`). Empty if the marker predates the
/// per-backend tracking or the id is no longer in the catalog.
backend_label: String,
}
/// Flags derived from filesystem state + config.
@@ -432,6 +440,12 @@ async fn compute_flags(
.map(|m| FailureBanner {
reason: m.reason,
failed_at: m.failed_at,
backend_label: m
.backend_id
.as_deref()
.and_then(crate::analyze::backend::find_backend)
.map(|b| b.label.clone())
.unwrap_or_default(),
})
} else {
None
@@ -556,8 +570,7 @@ pub async fn handle_my_cases(
// render. The common case is "nothing to do" and costs a handful of
// stat-calls per case; actual enqueues happen only when pre-conditions
// flip (new transcripts in, stale document, ...).
auto_trigger::try_enqueue_all_for_user(&user_root, &pipeline.analyze_tx, &settings, &events_tx)
.await;
auto_trigger::try_enqueue_all_for_user(&user_root, &pipeline.analyze_tx, &events_tx).await;
// Lazy retention sweep: at most one disk scan per /web/cases visit,
// reusing the user's retention policy from users.toml. Runs before
@@ -579,7 +592,7 @@ pub async fn handle_my_cases(
busy,
show_closed,
retention.auto_delete_days,
settings.llm_configured(),
crate::analyze::backend::any_backend_available(),
)
.await;
let total = cases.len();
@@ -669,7 +682,7 @@ pub async fn handle_case_page(
// Auto-analysis trigger for deep-link navigation: same evaluation as
// the list view. Document render below still wins the race if the
// worker happens to finish synchronously.
auto_trigger::try_enqueue(&case_dir, &pipeline.analyze_tx, &settings, &events_tx).await;
auto_trigger::try_enqueue(&case_dir, &pipeline.analyze_tx, &events_tx).await;
let case_id_str = case_id.to_string();
info!(slug = %user.slug, case_id = %case_id, "case page viewed");
@@ -687,7 +700,13 @@ pub async fn handle_case_page(
let (oneliner, oneliner_is_manual) = compute_oneliner_display(&case_dir, &recordings).await;
let a_busy = pipeline.analyze_busy.0.load(Ordering::Acquire);
let flags = compute_flags(&case_dir, &recordings, settings.llm_configured(), a_busy).await;
let flags = compute_flags(
&case_dir,
&recordings,
crate::analyze::backend::any_backend_available(),
a_busy,
)
.await;
let recordings_count = recordings.len();
let transcribed_count = recordings
@@ -747,6 +766,7 @@ pub async fn handle_case_page(
is_closed,
csrf_token: user.csrf_token,
debug_copy_json,
backends: crate::analyze::backend::available_backends(),
}
.render()
.map(Html)