feat: multi-backend LLM layer with per-request choice on case page

Replace the single [llm] block in settings.toml with a curated catalog
of LLM "backends" (provider + model + sampling params + system prompt)
in server/src/analyze/backend.rs. Two backends ship initially:
gpt_oss_120b (default, reasoning_effort=medium) and llama_3_1_405b
(uses response_format: json_schema to sidestep the <|eot_id|> leak).

The case page now renders one submit button per available backend; the
clicked button's name=backend value rides through AnalysisInput.backend_id
to the worker, which looks it up via find_backend() with default fallback.
A submitted unknown backend id is rejected as 400. .analysis_failed.json
records the failed backend and the failure banner surfaces its label.

The API key now comes from IONOS_API_KEY (env), not settings.toml; the
[llm] section is read into a discarded field so old configs still load.
Backends without a satisfied requires_api_key are filtered from the UI
(any_backend_available()). Three end-to-end tests that mocked the LLM
endpoint via settings.llm.url are #[ignore]'d with a clear note — they
need per-test backend injection (Arc<Vec<LlmBackend>> through the worker)
before they can be re-enabled.
This commit is contained in:
2026-05-03 14:57:21 +02:00
parent cbb072d0cc
commit 23ef84d9d8
18 changed files with 596 additions and 307 deletions
+10 -31
View File
@@ -1,16 +1,15 @@
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use tokio::io::AsyncWriteExt;
use tracing::{error, info, warn};
use super::backend::{default_backend, find_backend};
use super::{
ANALYSIS_INPUT_FILE, AnalysisInput, AnalyzeReceiver, DOCUMENT_FILE, auto_trigger, llm, prompt,
};
use crate::events::{self, CaseEventKind, EventSender};
use crate::gazetteer::Gazetteer;
use crate::settings::Settings;
use crate::{BusyGuard, WorkerBusy};
/// Consume analyze jobs sequentially. The external LLM tolerates parallelism,
@@ -18,26 +17,16 @@ use crate::{BusyGuard, WorkerBusy};
/// provider from bursts. Trivial to lift later via `tokio::spawn(process(..))`.
pub async fn run(
mut rx: AnalyzeReceiver,
settings: Arc<Settings>,
client: reqwest::Client,
worker_busy: WorkerBusy,
vocab: Arc<Gazetteer>,
events_tx: EventSender,
) {
info!(vocab_entries = vocab.len(), "Analyze worker started");
let timeout = Duration::from_secs(settings.llm.timeout_seconds);
while let Some(job) = rx.recv().await {
let _guard = BusyGuard::new(worker_busy.clone());
process(
&job.case_dir,
&settings,
&client,
&vocab,
timeout,
&events_tx,
)
.await;
process(&job.case_dir, &client, &vocab, &events_tx).await;
}
warn!("Analyze worker stopped (channel closed)");
@@ -45,10 +34,8 @@ pub async fn run(
async fn process(
case_dir: &Path,
settings: &Settings,
client: &reqwest::Client,
vocab: &Gazetteer,
timeout: Duration,
events_tx: &EventSender,
) {
let input_path = case_dir.join(ANALYSIS_INPUT_FILE);
@@ -83,6 +70,7 @@ async fn process(
case_dir,
&input.last_recording_mtime,
&format!("stub write: {e}"),
None,
)
.await;
return;
@@ -103,10 +91,13 @@ async fn process(
let user_content = prompt::render_prompt(&input);
let total_chars = user_content.chars().count();
let backend = find_backend(&input.backend_id).unwrap_or_else(default_backend);
info!(
case = %case_dir.display(),
recording_count = input.recordings.len(),
total_chars,
backend_id = %backend.id,
model = %backend.model_id,
"sending to llm"
);
@@ -114,30 +105,17 @@ async fn process(
// document write below, the recovery scan will re-enqueue this job and
// the call will be made again. Accepted cost — rare event, small
// per-call price. Response-caching in a `.tmp` file would prevent it.
let llm_settings = llm::LlmSettings {
base_url: &settings.llm.url,
api_key: &settings.llm.api_key,
model: &settings.llm.model,
temperature: settings.llm.temperature,
};
let document = match llm::chat_once(
client,
&llm_settings,
&settings.llm.system_prompt,
&user_content,
timeout,
)
.await
{
let document = match llm::chat_once(client, backend, &user_content).await {
Ok(text) => text,
Err(e) => {
// Do not log the response body — LlmError::Display is already
// redacted, but reinforce the rule here for future readers.
error!(case = %case_dir.display(), error = %e, "llm call failed");
error!(case = %case_dir.display(), backend_id = %backend.id, error = %e, "llm call failed");
let _ = auto_trigger::write_failure_marker(
case_dir,
&input.last_recording_mtime,
&format!("llm: {e}"),
Some(&backend.id),
)
.await;
return;
@@ -155,6 +133,7 @@ async fn process(
case_dir,
&input.last_recording_mtime,
&format!("write document: {e}"),
Some(&backend.id),
)
.await;
return;