diff --git a/server/src/analyze/backend.rs b/server/src/analyze/backend.rs index f02050d..71296ab 100644 --- a/server/src/analyze/backend.rs +++ b/server/src/analyze/backend.rs @@ -18,6 +18,27 @@ const DEFAULT_MAX_COMPLETION_TOKENS: u32 = 4096; const DEFAULT_TIMEOUT_SECONDS: u64 = 180; const DEFAULT_USE_JSON_SCHEMA: bool = true; +/// Llama 3.1 405B FP8 needs all three of: +/// 1. sampling (`temperature: 0.6`, `top_p: 0.9`) — without it the model +/// enters infinite repetition loops on prompts with conflicting +/// constraints (HuggingFace meta-llama/Llama-3.1-8B-Instruct disc. #32, +/// Ionos docs for the 405B model). +/// 2. `response_format: json_schema` — provides a hard server-side stop via +/// vLLM grammar-constrained decoding. +/// 3. an explicit format instruction telling the model what to put inside +/// `document` — without it the model emits `{"document": ""}` (silent +/// deletion, observed in test T4 on 2026-05-03). +const LLAMA_TEMPERATURE: f32 = 0.6; +const LLAMA_TOP_P: f32 = 0.9; + +/// Format instruction injected as a separate `system` message for backends +/// that aren't natively schema-aware. Kept as a Rust constant rather than a +/// prompt-file edit so the medical prompt stays purely about clinical rules. +const LLAMA_FORMAT_INSTRUCTION: &str = "AUSGABEFORMAT: Antworte ausschließlich \ + als JSON-Objekt der Form {\"document\": \"\"}. \ + Der Wert von \"document\" ist der Fließtext nach allen obigen Regeln. \ + Keine zusätzlichen Felder, kein Text außerhalb des JSON."; + /// Default system prompt for the consolidation LLM. Used by all backends /// unless overridden via `with_system_prompt`. /// @@ -52,10 +73,15 @@ pub struct LlmBackend { pub requires_api_key: bool, pub model_id: String, pub temperature: f32, + pub top_p: Option, pub max_completion_tokens: u32, pub reasoning_effort: Option, pub use_json_schema: bool, pub system_prompt: String, + /// Optional second `system` message appended after `system_prompt`. Used + /// to tell non-schema-aware models how to fill the schema's fields. `None` + /// means the body is sent unchanged (gpt-oss-style backends). + pub format_instruction: Option, pub timeout_seconds: u64, } @@ -73,6 +99,21 @@ impl LlmBackend { self.system_prompt = prompt.into(); self } + + fn with_temperature(mut self, temperature: f32) -> Self { + self.temperature = temperature; + self + } + + fn with_top_p(mut self, top_p: f32) -> Self { + self.top_p = Some(top_p); + self + } + + fn with_format_instruction(mut self, instruction: &str) -> Self { + self.format_instruction = Some(instruction.into()); + self + } } /// Ionos provider: shared endpoint, `IONOS_API_KEY` env var as bearer token. @@ -85,10 +126,12 @@ fn ionos_backend(id: &str, label: &str, model_id: &str) -> LlmBackend { requires_api_key: true, model_id: model_id.into(), temperature: DEFAULT_TEMPERATURE, + top_p: None, max_completion_tokens: DEFAULT_MAX_COMPLETION_TOKENS, reasoning_effort: None, use_json_schema: DEFAULT_USE_JSON_SCHEMA, system_prompt: DEFAULT_SYSTEM_PROMPT.into(), + format_instruction: None, timeout_seconds: DEFAULT_TIMEOUT_SECONDS, } } @@ -108,7 +151,10 @@ pub fn backends() -> &'static [LlmBackend] { "Llama 3.1 405B", "meta-llama/Meta-Llama-3.1-405B-Instruct-FP8", ) - .with_system_prompt(LLAMA_SYSTEM_PROMPT), + .with_system_prompt(LLAMA_SYSTEM_PROMPT) + .with_temperature(LLAMA_TEMPERATURE) + .with_top_p(LLAMA_TOP_P) + .with_format_instruction(LLAMA_FORMAT_INSTRUCTION), ] }) } @@ -204,10 +250,12 @@ mod tests { requires_api_key: true, model_id: "m".into(), temperature: 0.0, + top_p: None, max_completion_tokens: 1, reasoning_effort: None, use_json_schema: false, system_prompt: String::new(), + format_instruction: None, timeout_seconds: 1, }; assert!(with_key.is_available()); @@ -226,6 +274,37 @@ mod tests { assert!(no_auth_needed.is_available()); } + /// Llama 3.1 405B FP8 needs sampling + a format hint to be usable on + /// Ionos — verified empirically on 2026-05-03 (tests T1-T6 against the + /// real endpoint). Pin those values so a future refactor cannot silently + /// regress to the timeout-prone "greedy + no hint" config. + #[test] + fn llama_uses_sampling_and_format_instruction() { + let b = find_backend("llama_3_1_405b").unwrap(); + assert!((b.temperature - 0.6).abs() < f32::EPSILON); + assert_eq!(b.top_p, Some(0.9)); + assert!( + b.format_instruction.is_some(), + "Llama needs the format hint or it returns empty documents" + ); + assert!(b.use_json_schema, "Llama needs schema-enforced stop"); + } + + /// gpt-oss-120b is schema-aware out of the box. Asserting that it has + /// neither a top_p nor a format instruction guarantees the body sent to + /// Ionos stays byte-identical to pre-Llama-fix builds — the production + /// path that has been working for weeks must not silently change. + #[test] + fn gpt_oss_has_no_format_instruction_and_no_top_p() { + let b = find_backend("gpt_oss_120b").unwrap(); + assert!((b.temperature - 0.5).abs() < f32::EPSILON); + assert!(b.top_p.is_none(), "gpt-oss must not send top_p"); + assert!( + b.format_instruction.is_none(), + "gpt-oss must not get a second system message — body byte-stable" + ); + } + #[test] fn system_prompts_are_loaded() { for b in backends() { diff --git a/server/src/analyze/llm.rs b/server/src/analyze/llm.rs index fc4ebd3..cf22163 100644 --- a/server/src/analyze/llm.rs +++ b/server/src/analyze/llm.rs @@ -37,6 +37,8 @@ impl std::error::Error for LlmError {} struct ChatRequest<'a> { model: &'a str, temperature: f32, + #[serde(skip_serializing_if = "Option::is_none")] + top_p: Option, max_completion_tokens: u32, #[serde(skip_serializing_if = "Option::is_none")] reasoning_effort: Option<&'a str>, @@ -106,23 +108,36 @@ pub async fn chat_once( debug!(%url, model = %backend.model_id, backend = %backend.id, "calling llm chat completions"); let response_format = backend.use_json_schema.then(document_schema); + // Build messages dynamically: an optional second `system` message lets a + // backend (e.g. Llama 3.1) inject a format hint without touching the + // shared medical prompt. Backends without `format_instruction` produce + // exactly the two-message shape used pre-fix — gpt-oss path stays + // byte-identical. + let mut messages = Vec::with_capacity(3); + messages.push(Message { + role: "system", + content: &backend.system_prompt, + }); + if let Some(format) = backend.format_instruction.as_deref() { + messages.push(Message { + role: "system", + content: format, + }); + } + messages.push(Message { + role: "user", + content: user_content, + }); + let body = ChatRequest { model: &backend.model_id, temperature: backend.temperature, + top_p: backend.top_p, max_completion_tokens: backend.max_completion_tokens, reasoning_effort: backend.reasoning_effort.as_deref(), response_format, stream: false, - messages: vec![ - Message { - role: "system", - content: &backend.system_prompt, - }, - Message { - role: "user", - content: user_content, - }, - ], + messages, }; let timeout = Duration::from_secs(backend.timeout_seconds); diff --git a/server/src/analyze/worker.rs b/server/src/analyze/worker.rs index 506b695..b31a13f 100644 --- a/server/src/analyze/worker.rs +++ b/server/src/analyze/worker.rs @@ -127,6 +127,28 @@ async fn process( // `Amiodarone` back to `Amiodaron`). let document = vocab.replace(&document); + // Silent-deletion guard: schema-aware backends can return formally valid + // but empty envelopes (`{"document": ""}`) when the prompt is incompatible + // with the format constraint. Without this guard the worker would write + // an empty document.md, drop the failure marker, and the user would see + // a blank note with no diagnostic. Observed on Llama 3.1 405B FP8 + // (test T4, 2026-05-03) and previously on Mistral 24B. + if document.trim().is_empty() { + error!( + case = %case_dir.display(), + backend_id = %backend.id, + "llm returned empty document — silent deletion guard triggered" + ); + let _ = auto_trigger::write_failure_marker( + case_dir, + &input.last_recording_mtime, + "llm returned empty document", + Some(&backend.id), + ) + .await; + return; + } + if let Err(e) = write_atomic(&document_path, document.as_bytes()).await { error!(path = %document_path.display(), error = %e, "writing document failed"); let _ = auto_trigger::write_failure_marker(