Add Llama 3.1 405B specific LLM parameters

Configure Llama 3.1 405B with specific temperature, top_p, and a format
instruction. This addresses issues with infinite repetition and empty
document generation observed with this model. The format instruction is
added as a separate system message to avoid modifying the core medical
prompt.

server/src/analyze/backend.rs

@@ -18,6 +18,27 @@ const DEFAULT_MAX_COMPLETION_TOKENS: u32 = 4096;
 const DEFAULT_TIMEOUT_SECONDS: u64 = 180;
 const DEFAULT_USE_JSON_SCHEMA: bool = true;
 
+/// Llama 3.1 405B FP8 needs all three of:
+///   1. sampling (`temperature: 0.6`, `top_p: 0.9`) — without it the model
+///      enters infinite repetition loops on prompts with conflicting
+///      constraints (HuggingFace meta-llama/Llama-3.1-8B-Instruct disc. #32,
+///      Ionos docs for the 405B model).
+///   2. `response_format: json_schema` — provides a hard server-side stop via
+///      vLLM grammar-constrained decoding.
+///   3. an explicit format instruction telling the model what to put inside
+///      `document` — without it the model emits `{"document": ""}` (silent
+///      deletion, observed in test T4 on 2026-05-03).
+const LLAMA_TEMPERATURE: f32 = 0.6;
+const LLAMA_TOP_P: f32 = 0.9;
+
+/// Format instruction injected as a separate `system` message for backends
+/// that aren't natively schema-aware. Kept as a Rust constant rather than a
+/// prompt-file edit so the medical prompt stays purely about clinical rules.
+const LLAMA_FORMAT_INSTRUCTION: &str = "AUSGABEFORMAT: Antworte ausschließlich \
+    als JSON-Objekt der Form {\"document\": \"<konsolidierter Fließtext>\"}. \
+    Der Wert von \"document\" ist der Fließtext nach allen obigen Regeln. \
+    Keine zusätzlichen Felder, kein Text außerhalb des JSON.";
+
 /// Default system prompt for the consolidation LLM. Used by all backends
 /// unless overridden via `with_system_prompt`.
 ///
@@ -52,10 +73,15 @@ pub struct LlmBackend {
     pub requires_api_key: bool,
     pub model_id: String,
     pub temperature: f32,
+    pub top_p: Option<f32>,
     pub max_completion_tokens: u32,
     pub reasoning_effort: Option<String>,
     pub use_json_schema: bool,
     pub system_prompt: String,
+    /// Optional second `system` message appended after `system_prompt`. Used
+    /// to tell non-schema-aware models how to fill the schema's fields. `None`
+    /// means the body is sent unchanged (gpt-oss-style backends).
+    pub format_instruction: Option<String>,
     pub timeout_seconds: u64,
 }
 
@@ -73,6 +99,21 @@ impl LlmBackend {
         self.system_prompt = prompt.into();
         self
     }
+
+    fn with_temperature(mut self, temperature: f32) -> Self {
+        self.temperature = temperature;
+        self
+    }
+
+    fn with_top_p(mut self, top_p: f32) -> Self {
+        self.top_p = Some(top_p);
+        self
+    }
+
+    fn with_format_instruction(mut self, instruction: &str) -> Self {
+        self.format_instruction = Some(instruction.into());
+        self
+    }
 }
 
 /// Ionos provider: shared endpoint, `IONOS_API_KEY` env var as bearer token.
@@ -85,10 +126,12 @@ fn ionos_backend(id: &str, label: &str, model_id: &str) -> LlmBackend {
         requires_api_key: true,
         model_id: model_id.into(),
         temperature: DEFAULT_TEMPERATURE,
+        top_p: None,
         max_completion_tokens: DEFAULT_MAX_COMPLETION_TOKENS,
         reasoning_effort: None,
         use_json_schema: DEFAULT_USE_JSON_SCHEMA,
         system_prompt: DEFAULT_SYSTEM_PROMPT.into(),
+        format_instruction: None,
         timeout_seconds: DEFAULT_TIMEOUT_SECONDS,
     }
 }
@@ -108,7 +151,10 @@ pub fn backends() -> &'static [LlmBackend] {
                 "Llama 3.1 405B",
                 "meta-llama/Meta-Llama-3.1-405B-Instruct-FP8",
             )
-            .with_system_prompt(LLAMA_SYSTEM_PROMPT),
+            .with_system_prompt(LLAMA_SYSTEM_PROMPT)
+            .with_temperature(LLAMA_TEMPERATURE)
+            .with_top_p(LLAMA_TOP_P)
+            .with_format_instruction(LLAMA_FORMAT_INSTRUCTION),
         ]
     })
 }
@@ -204,10 +250,12 @@ mod tests {
             requires_api_key: true,
             model_id: "m".into(),
             temperature: 0.0,
+            top_p: None,
             max_completion_tokens: 1,
             reasoning_effort: None,
             use_json_schema: false,
             system_prompt: String::new(),
+            format_instruction: None,
             timeout_seconds: 1,
         };
         assert!(with_key.is_available());
@@ -226,6 +274,37 @@ mod tests {
         assert!(no_auth_needed.is_available());
     }
 
+    /// Llama 3.1 405B FP8 needs sampling + a format hint to be usable on
+    /// Ionos — verified empirically on 2026-05-03 (tests T1-T6 against the
+    /// real endpoint). Pin those values so a future refactor cannot silently
+    /// regress to the timeout-prone "greedy + no hint" config.
+    #[test]
+    fn llama_uses_sampling_and_format_instruction() {
+        let b = find_backend("llama_3_1_405b").unwrap();
+        assert!((b.temperature - 0.6).abs() < f32::EPSILON);
+        assert_eq!(b.top_p, Some(0.9));
+        assert!(
+            b.format_instruction.is_some(),
+            "Llama needs the format hint or it returns empty documents"
+        );
+        assert!(b.use_json_schema, "Llama needs schema-enforced stop");
+    }
+
+    /// gpt-oss-120b is schema-aware out of the box. Asserting that it has
+    /// neither a top_p nor a format instruction guarantees the body sent to
+    /// Ionos stays byte-identical to pre-Llama-fix builds — the production
+    /// path that has been working for weeks must not silently change.
+    #[test]
+    fn gpt_oss_has_no_format_instruction_and_no_top_p() {
+        let b = find_backend("gpt_oss_120b").unwrap();
+        assert!((b.temperature - 0.5).abs() < f32::EPSILON);
+        assert!(b.top_p.is_none(), "gpt-oss must not send top_p");
+        assert!(
+            b.format_instruction.is_none(),
+            "gpt-oss must not get a second system message — body byte-stable"
+        );
+    }
+
     #[test]
     fn system_prompts_are_loaded() {
         for b in backends() {

server/src/analyze/llm.rs

@@ -37,6 +37,8 @@ impl std::error::Error for LlmError {}
 struct ChatRequest<'a> {
     model: &'a str,
     temperature: f32,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    top_p: Option<f32>,
     max_completion_tokens: u32,
     #[serde(skip_serializing_if = "Option::is_none")]
     reasoning_effort: Option<&'a str>,
@@ -106,23 +108,36 @@ pub async fn chat_once(
     debug!(%url, model = %backend.model_id, backend = %backend.id, "calling llm chat completions");
 
     let response_format = backend.use_json_schema.then(document_schema);
+    // Build messages dynamically: an optional second `system` message lets a
+    // backend (e.g. Llama 3.1) inject a format hint without touching the
+    // shared medical prompt. Backends without `format_instruction` produce
+    // exactly the two-message shape used pre-fix — gpt-oss path stays
+    // byte-identical.
+    let mut messages = Vec::with_capacity(3);
+    messages.push(Message {
+        role: "system",
+        content: &backend.system_prompt,
+    });
+    if let Some(format) = backend.format_instruction.as_deref() {
+        messages.push(Message {
+            role: "system",
+            content: format,
+        });
+    }
+    messages.push(Message {
+        role: "user",
+        content: user_content,
+    });
+
     let body = ChatRequest {
         model: &backend.model_id,
         temperature: backend.temperature,
+        top_p: backend.top_p,
         max_completion_tokens: backend.max_completion_tokens,
         reasoning_effort: backend.reasoning_effort.as_deref(),
         response_format,
         stream: false,
-        messages: vec![
+        messages,
-            Message {
-                role: "system",
-                content: &backend.system_prompt,
-            },
-            Message {
-                role: "user",
-                content: user_content,
-            },
-        ],
     };
 
     let timeout = Duration::from_secs(backend.timeout_seconds);

server/src/analyze/worker.rs

@@ -127,6 +127,28 @@ async fn process(
     // `Amiodarone` back to `Amiodaron`).
     let document = vocab.replace(&document);
 
+    // Silent-deletion guard: schema-aware backends can return formally valid
+    // but empty envelopes (`{"document": ""}`) when the prompt is incompatible
+    // with the format constraint. Without this guard the worker would write
+    // an empty document.md, drop the failure marker, and the user would see
+    // a blank note with no diagnostic. Observed on Llama 3.1 405B FP8
+    // (test T4, 2026-05-03) and previously on Mistral 24B.
+    if document.trim().is_empty() {
+        error!(
+            case = %case_dir.display(),
+            backend_id = %backend.id,
+            "llm returned empty document — silent deletion guard triggered"
+        );
+        let _ = auto_trigger::write_failure_marker(
+            case_dir,
+            &input.last_recording_mtime,
+            "llm returned empty document",
+            Some(&backend.id),
+        )
+        .await;
+        return;
+    }
+
     if let Err(e) = write_atomic(&document_path, document.as_bytes()).await {
         error!(path = %document_path.display(), error = %e, "writing document failed");
         let _ = auto_trigger::write_failure_marker(