feat: multi-backend LLM layer with per-request choice on case page
Replace the single [llm] block in settings.toml with a curated catalog of LLM "backends" (provider + model + sampling params + system prompt) in server/src/analyze/backend.rs. Two backends ship initially: gpt_oss_120b (default, reasoning_effort=medium) and llama_3_1_405b (uses response_format: json_schema to sidestep the <|eot_id|> leak). The case page now renders one submit button per available backend; the clicked button's name=backend value rides through AnalysisInput.backend_id to the worker, which looks it up via find_backend() with default fallback. A submitted unknown backend id is rejected as 400. .analysis_failed.json records the failed backend and the failure banner surfaces its label. The API key now comes from IONOS_API_KEY (env), not settings.toml; the [llm] section is read into a discarded field so old configs still load. Backends without a satisfied requires_api_key are filtered from the UI (any_backend_available()). Three end-to-end tests that mocked the LLM endpoint via settings.llm.url are #[ignore]'d with a clear note — they need per-test backend injection (Arc<Vec<LlmBackend>> through the worker) before they can be re-enabled.
This commit is contained in:
@@ -1,16 +1,15 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
use super::backend::{default_backend, find_backend};
|
||||
use super::{
|
||||
ANALYSIS_INPUT_FILE, AnalysisInput, AnalyzeReceiver, DOCUMENT_FILE, auto_trigger, llm, prompt,
|
||||
};
|
||||
use crate::events::{self, CaseEventKind, EventSender};
|
||||
use crate::gazetteer::Gazetteer;
|
||||
use crate::settings::Settings;
|
||||
use crate::{BusyGuard, WorkerBusy};
|
||||
|
||||
/// Consume analyze jobs sequentially. The external LLM tolerates parallelism,
|
||||
@@ -18,26 +17,16 @@ use crate::{BusyGuard, WorkerBusy};
|
||||
/// provider from bursts. Trivial to lift later via `tokio::spawn(process(..))`.
|
||||
pub async fn run(
|
||||
mut rx: AnalyzeReceiver,
|
||||
settings: Arc<Settings>,
|
||||
client: reqwest::Client,
|
||||
worker_busy: WorkerBusy,
|
||||
vocab: Arc<Gazetteer>,
|
||||
events_tx: EventSender,
|
||||
) {
|
||||
info!(vocab_entries = vocab.len(), "Analyze worker started");
|
||||
let timeout = Duration::from_secs(settings.llm.timeout_seconds);
|
||||
|
||||
while let Some(job) = rx.recv().await {
|
||||
let _guard = BusyGuard::new(worker_busy.clone());
|
||||
process(
|
||||
&job.case_dir,
|
||||
&settings,
|
||||
&client,
|
||||
&vocab,
|
||||
timeout,
|
||||
&events_tx,
|
||||
)
|
||||
.await;
|
||||
process(&job.case_dir, &client, &vocab, &events_tx).await;
|
||||
}
|
||||
|
||||
warn!("Analyze worker stopped (channel closed)");
|
||||
@@ -45,10 +34,8 @@ pub async fn run(
|
||||
|
||||
async fn process(
|
||||
case_dir: &Path,
|
||||
settings: &Settings,
|
||||
client: &reqwest::Client,
|
||||
vocab: &Gazetteer,
|
||||
timeout: Duration,
|
||||
events_tx: &EventSender,
|
||||
) {
|
||||
let input_path = case_dir.join(ANALYSIS_INPUT_FILE);
|
||||
@@ -83,6 +70,7 @@ async fn process(
|
||||
case_dir,
|
||||
&input.last_recording_mtime,
|
||||
&format!("stub write: {e}"),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
@@ -103,10 +91,13 @@ async fn process(
|
||||
|
||||
let user_content = prompt::render_prompt(&input);
|
||||
let total_chars = user_content.chars().count();
|
||||
let backend = find_backend(&input.backend_id).unwrap_or_else(default_backend);
|
||||
info!(
|
||||
case = %case_dir.display(),
|
||||
recording_count = input.recordings.len(),
|
||||
total_chars,
|
||||
backend_id = %backend.id,
|
||||
model = %backend.model_id,
|
||||
"sending to llm"
|
||||
);
|
||||
|
||||
@@ -114,30 +105,17 @@ async fn process(
|
||||
// document write below, the recovery scan will re-enqueue this job and
|
||||
// the call will be made again. Accepted cost — rare event, small
|
||||
// per-call price. Response-caching in a `.tmp` file would prevent it.
|
||||
let llm_settings = llm::LlmSettings {
|
||||
base_url: &settings.llm.url,
|
||||
api_key: &settings.llm.api_key,
|
||||
model: &settings.llm.model,
|
||||
temperature: settings.llm.temperature,
|
||||
};
|
||||
let document = match llm::chat_once(
|
||||
client,
|
||||
&llm_settings,
|
||||
&settings.llm.system_prompt,
|
||||
&user_content,
|
||||
timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
let document = match llm::chat_once(client, backend, &user_content).await {
|
||||
Ok(text) => text,
|
||||
Err(e) => {
|
||||
// Do not log the response body — LlmError::Display is already
|
||||
// redacted, but reinforce the rule here for future readers.
|
||||
error!(case = %case_dir.display(), error = %e, "llm call failed");
|
||||
error!(case = %case_dir.display(), backend_id = %backend.id, error = %e, "llm call failed");
|
||||
let _ = auto_trigger::write_failure_marker(
|
||||
case_dir,
|
||||
&input.last_recording_mtime,
|
||||
&format!("llm: {e}"),
|
||||
Some(&backend.id),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
@@ -155,6 +133,7 @@ async fn process(
|
||||
case_dir,
|
||||
&input.last_recording_mtime,
|
||||
&format!("write document: {e}"),
|
||||
Some(&backend.id),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
|
||||
Reference in New Issue
Block a user