feat: multi-backend LLM layer with per-request choice on case page
Replace the single [llm] block in settings.toml with a curated catalog of LLM "backends" (provider + model + sampling params + system prompt) in server/src/analyze/backend.rs. Two backends ship initially: gpt_oss_120b (default, reasoning_effort=medium) and llama_3_1_405b (uses response_format: json_schema to sidestep the <|eot_id|> leak). The case page now renders one submit button per available backend; the clicked button's name=backend value rides through AnalysisInput.backend_id to the worker, which looks it up via find_backend() with default fallback. A submitted unknown backend id is rejected as 400. .analysis_failed.json records the failed backend and the failure banner surfaces its label. The API key now comes from IONOS_API_KEY (env), not settings.toml; the [llm] section is read into a discarded field so old configs still load. Backends without a satisfied requires_api_key are filtered from the UI (any_backend_available()). Three end-to-end tests that mocked the LLM endpoint via settings.llm.url are #[ignore]'d with a clear note — they need per-test backend injection (Arc<Vec<LlmBackend>> through the worker) before they can be re-enabled.
This commit is contained in:
@@ -14,6 +14,7 @@ use tracing::{info, warn};
|
||||
use doctate_common::timestamp::{filename_stem_to_recorded_at, now_rfc3339};
|
||||
use doctate_common::{RECORDING_META_SUFFIX, TranscriptState};
|
||||
|
||||
use crate::analyze::backend::{any_backend_available, find_backend};
|
||||
use crate::analyze::{
|
||||
ANALYSIS_INPUT_FILE, AnalysisInput, AnalyzeJob, AnalyzeSender, DOCUMENT_FILE, RecordingInput,
|
||||
};
|
||||
@@ -30,7 +31,25 @@ use crate::paths::{
|
||||
};
|
||||
use crate::routes::user_web::locate_case_or_404;
|
||||
use crate::routes::web::validate_filename;
|
||||
use crate::settings::Settings;
|
||||
|
||||
/// Form body for `POST /web/cases/{id}/analyze`. The `backend` value is set
|
||||
/// by the clicked submit button (`<button name="backend" value="…">`); an
|
||||
/// empty value falls back to the default backend.
|
||||
#[derive(Deserialize)]
|
||||
pub struct AnalyzeForm {
|
||||
#[serde(default)]
|
||||
pub csrf_token: String,
|
||||
#[serde(default)]
|
||||
pub backend: String,
|
||||
#[serde(default)]
|
||||
pub return_to: Option<String>,
|
||||
}
|
||||
|
||||
impl HasCsrfToken for AnalyzeForm {
|
||||
fn csrf_token(&self) -> &str {
|
||||
&self.csrf_token
|
||||
}
|
||||
}
|
||||
|
||||
/// POST /web/cases/{case_id}/analyze
|
||||
///
|
||||
@@ -42,18 +61,29 @@ use crate::settings::Settings;
|
||||
pub async fn handle_analyze_case(
|
||||
user: AuthenticatedWebUser,
|
||||
State(config): State<Arc<Config>>,
|
||||
State(settings): State<Arc<Settings>>,
|
||||
State(analyze_tx): State<AnalyzeSender>,
|
||||
State(events_tx): State<EventSender>,
|
||||
CaseIdPath(case_id): CaseIdPath,
|
||||
CsrfForm(form): CsrfForm<CsrfOnlyForm>,
|
||||
CsrfForm(form): CsrfForm<AnalyzeForm>,
|
||||
) -> Result<Redirect, AppError> {
|
||||
if !settings.llm_configured() {
|
||||
if !any_backend_available() {
|
||||
return Err(AppError::ServiceUnavailable(
|
||||
"LLM-Analyse nicht konfiguriert".into(),
|
||||
));
|
||||
}
|
||||
|
||||
// Resolve the requested backend. Empty string falls back to the default
|
||||
// backend; an unknown id is rejected as a 400 so the user notices the
|
||||
// misconfiguration instead of silently getting a different model.
|
||||
let backend_id = if form.backend.is_empty() {
|
||||
crate::analyze::backend::default_backend().id.clone()
|
||||
} else {
|
||||
find_backend(&form.backend)
|
||||
.ok_or_else(|| AppError::BadRequest(format!("Unbekanntes Backend: {}", form.backend)))?
|
||||
.id
|
||||
.clone()
|
||||
};
|
||||
|
||||
let user_root = config.data_path.join(&user.slug);
|
||||
let case_dir = locate_case_or_404(&user_root, &case_id, &user.slug, "analyze").await?;
|
||||
|
||||
@@ -67,7 +97,7 @@ pub async fn handle_analyze_case(
|
||||
Err(e) => return Err(AppError::Internal(format!("remove old document: {e}"))),
|
||||
}
|
||||
|
||||
let input = build_analysis_input(&case_dir).await?;
|
||||
let input = build_analysis_input(&case_dir, &backend_id).await?;
|
||||
let input_path = case_dir.join(ANALYSIS_INPUT_FILE);
|
||||
write_input_create_new(&input_path, &input).await?;
|
||||
|
||||
@@ -174,7 +204,13 @@ fn resolve_list_return_path(headers: &HeaderMap) -> String {
|
||||
/// `<stem>.json` for each, skips blank transcripts, and computes
|
||||
/// `last_recording_mtime` as the max mtime over ALL `.m4a` (including blank
|
||||
/// ones — a late blank addendum still counts as activity).
|
||||
pub(crate) async fn build_analysis_input(case_dir: &Path) -> Result<AnalysisInput, AppError> {
|
||||
///
|
||||
/// `backend_id` is stored as-is; an empty string lets the worker fall back
|
||||
/// to the default backend at dequeue time.
|
||||
pub(crate) async fn build_analysis_input(
|
||||
case_dir: &Path,
|
||||
backend_id: &str,
|
||||
) -> Result<AnalysisInput, AppError> {
|
||||
let m4as = collect_m4as(case_dir).await?;
|
||||
if m4as.is_empty() {
|
||||
return Err(AppError::BadRequest("Keine Aufnahmen im Fall".into()));
|
||||
@@ -196,6 +232,7 @@ pub(crate) async fn build_analysis_input(case_dir: &Path) -> Result<AnalysisInpu
|
||||
Ok(AnalysisInput {
|
||||
last_recording_mtime,
|
||||
recordings,
|
||||
backend_id: backend_id.to_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user