feat: multi-backend LLM layer with per-request choice on case page
Replace the single [llm] block in settings.toml with a curated catalog of LLM "backends" (provider + model + sampling params + system prompt) in server/src/analyze/backend.rs. Two backends ship initially: gpt_oss_120b (default, reasoning_effort=medium) and llama_3_1_405b (uses response_format: json_schema to sidestep the <|eot_id|> leak). The case page now renders one submit button per available backend; the clicked button's name=backend value rides through AnalysisInput.backend_id to the worker, which looks it up via find_backend() with default fallback. A submitted unknown backend id is rejected as 400. .analysis_failed.json records the failed backend and the failure banner surfaces its label. The API key now comes from IONOS_API_KEY (env), not settings.toml; the [llm] section is read into a discarded field so old configs still load. Backends without a satisfied requires_api_key are filtered from the UI (any_backend_available()). Three end-to-end tests that mocked the LLM endpoint via settings.llm.url are #[ignore]'d with a clear note — they need per-test backend injection (Arc<Vec<LlmBackend>> through the worker) before they can be re-enabled.
This commit is contained in:
@@ -9,7 +9,6 @@
|
||||
//! `try_enqueue` wrapper performs the actual side effects.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -22,7 +21,6 @@ use super::{ANALYSIS_INPUT_FILE, AnalysisInput, AnalyzeJob, AnalyzeSender, DOCUM
|
||||
use crate::events::{self, CaseEventKind, EventSender};
|
||||
use crate::paths;
|
||||
use crate::routes::case_actions::build_analysis_input;
|
||||
use crate::settings::Settings;
|
||||
|
||||
/// Filename of the per-case failure marker. Written by the worker on
|
||||
/// error, deleted on success. Presence with a matching
|
||||
@@ -37,6 +35,10 @@ pub struct FailureMarker {
|
||||
pub last_recording_mtime: String,
|
||||
pub reason: String,
|
||||
pub failed_at: String,
|
||||
/// Id of the backend that failed (when known). Old markers without
|
||||
/// the field deserialize to `None`.
|
||||
#[serde(default)]
|
||||
pub backend_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Outcome of the pre-condition check. `Skip` carries a static string so
|
||||
@@ -96,22 +98,19 @@ pub async fn evaluate_case(case_dir: &Path) -> AutoDecision {
|
||||
/// Evaluate and, on `Enqueue`, prepare and send the job. Returns `true`
|
||||
/// iff a job was actually sent. Silent on routine skips; `warn!` only on
|
||||
/// actual failures (build_input, write, send).
|
||||
pub async fn try_enqueue(
|
||||
case_dir: &Path,
|
||||
tx: &AnalyzeSender,
|
||||
settings: &Arc<Settings>,
|
||||
events_tx: &EventSender,
|
||||
) -> bool {
|
||||
if !settings.llm_configured() {
|
||||
return false;
|
||||
}
|
||||
|
||||
///
|
||||
/// LLM-availability is *not* checked here — callers (web handlers) check
|
||||
/// via `analyze::backend::any_backend_available()` before deciding to
|
||||
/// render the analyze affordance at all. If the worker dequeues a job
|
||||
/// without an available backend it will fail at the LLM call and write a
|
||||
/// failure marker, which is graceful degradation rather than data loss.
|
||||
pub async fn try_enqueue(case_dir: &Path, tx: &AnalyzeSender, events_tx: &EventSender) -> bool {
|
||||
match evaluate_case(case_dir).await {
|
||||
AutoDecision::Skip(_) => return false,
|
||||
AutoDecision::Enqueue => {}
|
||||
}
|
||||
|
||||
let input = match build_analysis_input(case_dir).await {
|
||||
let input = match build_analysis_input(case_dir, "").await {
|
||||
Ok(i) => i,
|
||||
Err(e) => {
|
||||
warn!(case = %case_dir.display(), error = ?e, "auto-analyze: build input failed");
|
||||
@@ -159,7 +158,6 @@ pub async fn try_enqueue(
|
||||
pub async fn try_enqueue_all_for_user(
|
||||
user_root: &Path,
|
||||
tx: &AnalyzeSender,
|
||||
settings: &Arc<Settings>,
|
||||
events_tx: &EventSender,
|
||||
) {
|
||||
let Ok(mut entries) = tokio::fs::read_dir(user_root).await else {
|
||||
@@ -179,7 +177,7 @@ pub async fn try_enqueue_all_for_user(
|
||||
if crate::paths::is_closed(&case_path).await {
|
||||
continue;
|
||||
}
|
||||
try_enqueue(&case_path, tx, settings, events_tx).await;
|
||||
try_enqueue(&case_path, tx, events_tx).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,11 +189,13 @@ pub async fn write_failure_marker(
|
||||
case_dir: &Path,
|
||||
last_recording_mtime: &str,
|
||||
reason: &str,
|
||||
backend_id: Option<&str>,
|
||||
) -> std::io::Result<()> {
|
||||
let marker = FailureMarker {
|
||||
last_recording_mtime: last_recording_mtime.to_owned(),
|
||||
reason: reason.to_owned(),
|
||||
failed_at: doctate_common::now_rfc3339(),
|
||||
backend_id: backend_id.map(|s| s.to_owned()),
|
||||
};
|
||||
let bytes = serde_json::to_vec_pretty(&marker)
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
@@ -443,6 +443,7 @@ mod tests {
|
||||
last_recording_mtime: rfc3339_of(base),
|
||||
reason: "test".into(),
|
||||
failed_at: "2026-01-01T10-05-00Z".into(),
|
||||
backend_id: None,
|
||||
};
|
||||
fs::write(
|
||||
dir.path().join(FAILURE_MARKER_FILE),
|
||||
@@ -472,6 +473,7 @@ mod tests {
|
||||
last_recording_mtime: rfc3339_of(base - Duration::from_secs(3600)),
|
||||
reason: "test".into(),
|
||||
failed_at: "2025-12-31T23-00-00Z".into(),
|
||||
backend_id: None,
|
||||
};
|
||||
fs::write(
|
||||
dir.path().join(FAILURE_MARKER_FILE),
|
||||
@@ -498,7 +500,6 @@ mod tests {
|
||||
async fn try_enqueue_all_sends_only_eligible_cases() {
|
||||
use crate::analyze::channel as analyze_channel;
|
||||
use crate::events::channel as events_channel;
|
||||
use crate::settings::Settings;
|
||||
|
||||
let user_root = TempDir::new().unwrap();
|
||||
|
||||
@@ -539,12 +540,8 @@ mod tests {
|
||||
|
||||
let (tx, mut rx) = analyze_channel();
|
||||
let events_tx = events_channel();
|
||||
let mut settings = Settings::default();
|
||||
settings.llm.url = "http://localhost:9999".into();
|
||||
settings.llm.model = "test".into();
|
||||
let settings = Arc::new(settings);
|
||||
|
||||
try_enqueue_all_for_user(user_root.path(), &tx, &settings, &events_tx).await;
|
||||
try_enqueue_all_for_user(user_root.path(), &tx, &events_tx).await;
|
||||
|
||||
// Drop the sender half so recv closes after draining.
|
||||
drop(tx);
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
//! LLM backend catalog.
|
||||
//!
|
||||
//! A "backend" is a fully self-describing profile: model + sampling params +
|
||||
//! system prompt + endpoint + auth. Multiple profiles per model are allowed
|
||||
//! (e.g. several Llama-3.1 variants with different temperatures).
|
||||
//!
|
||||
//! Provider constructors (`ionos_backend`, future `ollama_backend`) own the
|
||||
//! "auth + endpoint" knowledge for their family; only globally-shared values
|
||||
//! (temperature, max_completion_tokens, timeout, use_json_schema, default
|
||||
//! prompt) live as module-level `const`s. Adding a new profile within an
|
||||
//! existing provider is a one-line `vec!` entry; adding a new provider is a
|
||||
//! new constructor function.
|
||||
|
||||
use std::sync::OnceLock;
|
||||
|
||||
const DEFAULT_TEMPERATURE: f32 = 0.5;
|
||||
const DEFAULT_MAX_COMPLETION_TOKENS: u32 = 4096;
|
||||
const DEFAULT_TIMEOUT_SECONDS: u64 = 180;
|
||||
const DEFAULT_USE_JSON_SCHEMA: bool = true;
|
||||
|
||||
/// Default system prompt for the consolidation LLM. Used by all backends
|
||||
/// unless overridden via `with_system_prompt`.
|
||||
///
|
||||
/// Sandbox-validated against case `c414cf52` (3 runs each, fair pipeline
|
||||
/// with pre- and post-LLM gazetteer pass against the current vocab):
|
||||
/// - eliminates two hallucination classes the previous prompt produced —
|
||||
/// unmarked "Hypotonie" when the dictation said "Hypertonie" (0/3 vs 2/3),
|
||||
/// and inventing a unit ("35 ng/l") for a dictated value without unit
|
||||
/// (0/3 vs 2/3)
|
||||
/// - keeps Latin terms intact ("Punctum Maximum", "Apex Cordis", "Axilla"
|
||||
/// in 3/3 vs 2/3)
|
||||
/// - relies on the gazetteer for drug-name correction (Enoxaparin etc.) —
|
||||
/// the prompt no longer needs a separate "AKTIVE KORREKTUR" hammer-block
|
||||
/// because the gazetteer's pre-LLM pass repairs typical typos before the
|
||||
/// LLM ever sees them
|
||||
/// - block layout: AUFGABE / QUELLE / KORREKTUR-POLITIK / TREUE / CHRONOLOGIE
|
||||
/// / DOSIERUNGSSCHEMA / MARKIERUNGEN — each rule appears exactly once
|
||||
const DEFAULT_SYSTEM_PROMPT: &str = include_str!("../../prompts/default_system_prompt.md");
|
||||
|
||||
/// Llama-specific system prompt. Currently identical to the default; kept
|
||||
/// as a separate file so adjustments for Llama-3.1's behaviour (e.g.
|
||||
/// stricter "do not hallucinate" wording) can be tuned independently of
|
||||
/// gpt-oss without forking the default for everyone.
|
||||
const LLAMA_SYSTEM_PROMPT: &str = include_str!("../../prompts/llama_system_prompt.md");
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct LlmBackend {
|
||||
pub id: String,
|
||||
pub label: String,
|
||||
pub url: String,
|
||||
pub api_key: String,
|
||||
pub requires_api_key: bool,
|
||||
pub model_id: String,
|
||||
pub temperature: f32,
|
||||
pub max_completion_tokens: u32,
|
||||
pub reasoning_effort: Option<String>,
|
||||
pub use_json_schema: bool,
|
||||
pub system_prompt: String,
|
||||
pub timeout_seconds: u64,
|
||||
}
|
||||
|
||||
impl LlmBackend {
|
||||
pub fn is_available(&self) -> bool {
|
||||
!self.requires_api_key || !self.api_key.is_empty()
|
||||
}
|
||||
|
||||
fn with_reasoning_effort(mut self, effort: &str) -> Self {
|
||||
self.reasoning_effort = Some(effort.into());
|
||||
self
|
||||
}
|
||||
|
||||
fn with_system_prompt(mut self, prompt: &str) -> Self {
|
||||
self.system_prompt = prompt.into();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Ionos provider: shared endpoint, `IONOS_API_KEY` env var as bearer token.
|
||||
fn ionos_backend(id: &str, label: &str, model_id: &str) -> LlmBackend {
|
||||
LlmBackend {
|
||||
id: id.into(),
|
||||
label: label.into(),
|
||||
url: "https://openai.inference.de-txl.ionos.com".into(),
|
||||
api_key: std::env::var("IONOS_API_KEY").unwrap_or_default(),
|
||||
requires_api_key: true,
|
||||
model_id: model_id.into(),
|
||||
temperature: DEFAULT_TEMPERATURE,
|
||||
max_completion_tokens: DEFAULT_MAX_COMPLETION_TOKENS,
|
||||
reasoning_effort: None,
|
||||
use_json_schema: DEFAULT_USE_JSON_SCHEMA,
|
||||
system_prompt: DEFAULT_SYSTEM_PROMPT.into(),
|
||||
timeout_seconds: DEFAULT_TIMEOUT_SECONDS,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the curated catalog of backends. Lazy-initialized once on first
|
||||
/// access; the env-var read for `api_key` happens here, so missing
|
||||
/// `IONOS_API_KEY` at startup yields backends with empty keys (filtered out
|
||||
/// by `available_backends()`).
|
||||
pub fn backends() -> &'static [LlmBackend] {
|
||||
static B: OnceLock<Vec<LlmBackend>> = OnceLock::new();
|
||||
B.get_or_init(|| {
|
||||
vec![
|
||||
ionos_backend("gpt_oss_120b", "GPT OSS 120b", "openai/gpt-oss-120b")
|
||||
.with_reasoning_effort("medium"),
|
||||
ionos_backend(
|
||||
"llama_3_1_405b",
|
||||
"Llama 3.1 405B",
|
||||
"meta-llama/Meta-Llama-3.1-405B-Instruct-FP8",
|
||||
)
|
||||
.with_system_prompt(LLAMA_SYSTEM_PROMPT),
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
pub fn find_backend(id: &str) -> Option<&'static LlmBackend> {
|
||||
backends().iter().find(|b| b.id == id)
|
||||
}
|
||||
|
||||
pub fn default_backend() -> &'static LlmBackend {
|
||||
&backends()[0]
|
||||
}
|
||||
|
||||
pub fn available_backends() -> Vec<&'static LlmBackend> {
|
||||
backends().iter().filter(|b| b.is_available()).collect()
|
||||
}
|
||||
|
||||
pub fn any_backend_available() -> bool {
|
||||
backends().iter().any(|b| b.is_available())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashSet;
|
||||
|
||||
/// The default backend is the first entry. Anything else would silently
|
||||
/// shift production traffic when entries are reordered.
|
||||
#[test]
|
||||
fn default_backend_is_first_in_catalog() {
|
||||
assert_eq!(default_backend().id, backends()[0].id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_backend_is_gpt_oss_120b() {
|
||||
assert_eq!(default_backend().id, "gpt_oss_120b");
|
||||
assert_eq!(default_backend().model_id, "openai/gpt-oss-120b");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_backend_returns_known_id() {
|
||||
assert!(find_backend("gpt_oss_120b").is_some());
|
||||
assert!(find_backend("llama_3_1_405b").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_backend_returns_none_for_unknown_id() {
|
||||
assert!(find_backend("does-not-exist").is_none());
|
||||
assert!(find_backend("").is_none());
|
||||
}
|
||||
|
||||
/// Catalog ids are routed via stringly-typed lookup; duplicates would
|
||||
/// turn `find_backend` non-deterministic in the face of reorderings.
|
||||
#[test]
|
||||
fn catalog_ids_are_unique() {
|
||||
let mut seen: HashSet<&str> = HashSet::new();
|
||||
for b in backends() {
|
||||
assert!(seen.insert(b.id.as_str()), "duplicate id: {}", b.id);
|
||||
}
|
||||
}
|
||||
|
||||
/// gpt-oss is a reasoning model, llama 3.1 is not. Asserting both keeps
|
||||
/// the conditional `reasoning_effort` field in `chat_once` correct.
|
||||
#[test]
|
||||
fn reasoning_effort_is_set_per_model_family() {
|
||||
assert_eq!(
|
||||
find_backend("gpt_oss_120b").unwrap().reasoning_effort,
|
||||
Some("medium".to_string())
|
||||
);
|
||||
assert!(
|
||||
find_backend("llama_3_1_405b")
|
||||
.unwrap()
|
||||
.reasoning_effort
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ionos_backends_share_endpoint_and_require_api_key() {
|
||||
for id in ["gpt_oss_120b", "llama_3_1_405b"] {
|
||||
let b = find_backend(id).unwrap();
|
||||
assert_eq!(b.url, "https://openai.inference.de-txl.ionos.com");
|
||||
assert!(b.requires_api_key, "{id} should require api key");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_available_requires_filled_api_key_when_required() {
|
||||
let with_key = LlmBackend {
|
||||
id: "x".into(),
|
||||
label: "x".into(),
|
||||
url: "https://x".into(),
|
||||
api_key: "k".into(),
|
||||
requires_api_key: true,
|
||||
model_id: "m".into(),
|
||||
temperature: 0.0,
|
||||
max_completion_tokens: 1,
|
||||
reasoning_effort: None,
|
||||
use_json_schema: false,
|
||||
system_prompt: String::new(),
|
||||
timeout_seconds: 1,
|
||||
};
|
||||
assert!(with_key.is_available());
|
||||
|
||||
let no_key = LlmBackend {
|
||||
api_key: String::new(),
|
||||
..with_key
|
||||
};
|
||||
assert!(!no_key.is_available());
|
||||
|
||||
let no_auth_needed = LlmBackend {
|
||||
api_key: String::new(),
|
||||
requires_api_key: false,
|
||||
..no_key
|
||||
};
|
||||
assert!(no_auth_needed.is_available());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_prompts_are_loaded() {
|
||||
for b in backends() {
|
||||
assert!(
|
||||
!b.system_prompt.trim().is_empty(),
|
||||
"{} has an empty system prompt",
|
||||
b.id
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
+60
-35
@@ -2,8 +2,11 @@ use std::time::Duration;
|
||||
|
||||
use doctate_common::join_url;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::json;
|
||||
use tracing::debug;
|
||||
|
||||
use super::backend::LlmBackend;
|
||||
|
||||
/// Error variants returned by the LLM client (OpenAI-compatible Chat
|
||||
/// Completions API — Ionos, Azure OpenAI, OpenAI proper, Together.ai, etc.).
|
||||
///
|
||||
@@ -13,14 +16,8 @@ use tracing::debug;
|
||||
/// the status and a short category string are safe to log.
|
||||
#[derive(Debug)]
|
||||
pub enum LlmError {
|
||||
/// Transport-level failure (DNS, TLS, connect, timeout). `reqwest::Error`
|
||||
/// is intentionally included — its `Display` only reveals URL/method,
|
||||
/// not request headers, so the API key stays out of logs.
|
||||
Http(reqwest::Error),
|
||||
/// Provider returned a non-2xx status. Body is deliberately discarded.
|
||||
Status { status: u16 },
|
||||
/// Response was 2xx but the expected `choices[0].message.content`
|
||||
/// shape was absent or empty.
|
||||
Parse(&'static str),
|
||||
}
|
||||
|
||||
@@ -36,22 +33,15 @@ impl std::fmt::Display for LlmError {
|
||||
|
||||
impl std::error::Error for LlmError {}
|
||||
|
||||
/// Grouped provider configuration. Passed by reference to `chat_once`.
|
||||
pub struct LlmSettings<'a> {
|
||||
pub base_url: &'a str,
|
||||
pub api_key: &'a str,
|
||||
pub model: &'a str,
|
||||
pub temperature: f32,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ChatRequest<'a> {
|
||||
model: &'a str,
|
||||
temperature: f32,
|
||||
/// Tunes reasoning depth for gpt-oss reasoning models (Ionos):
|
||||
/// `low`/`medium`/`high` trade off hallucination risk against latency and
|
||||
/// token cost. Ignored by non-reasoning OpenAI-compatible models.
|
||||
reasoning_effort: &'static str,
|
||||
max_completion_tokens: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
reasoning_effort: Option<&'a str>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
response_format: Option<serde_json::Value>,
|
||||
stream: bool,
|
||||
messages: Vec<Message<'a>>,
|
||||
}
|
||||
@@ -77,30 +67,56 @@ struct ResponseMessage {
|
||||
content: Option<String>,
|
||||
}
|
||||
|
||||
/// Single-shot chat completion against an OpenAI-compatible endpoint.
|
||||
/// The `api_key` in `settings` is used as a Bearer token; do not log it.
|
||||
/// An empty `api_key` is treated as "no auth" — the `Authorization` header
|
||||
/// is omitted entirely, which is the correct mode for Ollama's
|
||||
/// OpenAI-compatible endpoint (some gateways object to a blank Bearer).
|
||||
#[derive(Deserialize)]
|
||||
struct DocumentEnvelope {
|
||||
document: String,
|
||||
}
|
||||
|
||||
/// JSON schema used by `use_json_schema` backends. Forces the model to emit
|
||||
/// `{"document": "..."}` so decoding stops at the closing brace — the same
|
||||
/// code path works for gpt-oss, Llama 3.1 (where this also avoids the
|
||||
/// `<|eot_id|>` leak), and any future schema-aware OpenAI-compatible model.
|
||||
fn document_schema() -> serde_json::Value {
|
||||
json!({
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "document",
|
||||
"strict": true,
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"document": { "type": "string" }
|
||||
},
|
||||
"required": ["document"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Single-shot chat completion against the OpenAI-compatible endpoint
|
||||
/// described by `backend`. Empty `api_key` omits the `Authorization`
|
||||
/// header (correct for unauthenticated local backends like Ollama).
|
||||
pub async fn chat_once(
|
||||
client: &reqwest::Client,
|
||||
settings: &LlmSettings<'_>,
|
||||
system_prompt: &str,
|
||||
backend: &LlmBackend,
|
||||
user_content: &str,
|
||||
timeout: Duration,
|
||||
) -> Result<String, LlmError> {
|
||||
let url = join_url(settings.base_url, "/v1/chat/completions");
|
||||
debug!(%url, model = settings.model, "calling llm chat completions");
|
||||
let url = join_url(&backend.url, "/v1/chat/completions");
|
||||
debug!(%url, model = %backend.model_id, backend = %backend.id, "calling llm chat completions");
|
||||
|
||||
let response_format = backend.use_json_schema.then(document_schema);
|
||||
let body = ChatRequest {
|
||||
model: settings.model,
|
||||
temperature: settings.temperature,
|
||||
reasoning_effort: "medium",
|
||||
model: &backend.model_id,
|
||||
temperature: backend.temperature,
|
||||
max_completion_tokens: backend.max_completion_tokens,
|
||||
reasoning_effort: backend.reasoning_effort.as_deref(),
|
||||
response_format,
|
||||
stream: false,
|
||||
messages: vec![
|
||||
Message {
|
||||
role: "system",
|
||||
content: system_prompt,
|
||||
content: &backend.system_prompt,
|
||||
},
|
||||
Message {
|
||||
role: "user",
|
||||
@@ -109,9 +125,10 @@ pub async fn chat_once(
|
||||
],
|
||||
};
|
||||
|
||||
let timeout = Duration::from_secs(backend.timeout_seconds);
|
||||
let mut req = client.post(&url).timeout(timeout).json(&body);
|
||||
if !settings.api_key.is_empty() {
|
||||
req = req.bearer_auth(settings.api_key);
|
||||
if !backend.api_key.is_empty() {
|
||||
req = req.bearer_auth(&backend.api_key);
|
||||
}
|
||||
let response = req.send().await.map_err(LlmError::Http)?;
|
||||
|
||||
@@ -132,7 +149,15 @@ pub async fn chat_once(
|
||||
.and_then(|c| c.message.content)
|
||||
.ok_or(LlmError::Parse("missing choices[0].message.content"))?;
|
||||
|
||||
let trimmed = content.trim().to_string();
|
||||
let text = if backend.use_json_schema {
|
||||
let env: DocumentEnvelope = serde_json::from_str(&content)
|
||||
.map_err(|_| LlmError::Parse("schema response was not {document: string}"))?;
|
||||
env.document
|
||||
} else {
|
||||
content
|
||||
};
|
||||
|
||||
let trimmed = text.trim().to_string();
|
||||
if trimmed.is_empty() {
|
||||
return Err(LlmError::Parse("empty content after trim"));
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
pub mod auto_trigger;
|
||||
pub mod backend;
|
||||
pub mod llm;
|
||||
pub mod prompt;
|
||||
pub mod recovery;
|
||||
@@ -41,6 +42,10 @@ pub struct AnalysisInput {
|
||||
/// diagnostics / future "Nachtrag" detection.
|
||||
pub last_recording_mtime: String,
|
||||
pub recordings: Vec<RecordingInput>,
|
||||
/// Selected LLM backend id; empty string falls back to the default
|
||||
/// backend. Old inputs without the field deserialize to `""`.
|
||||
#[serde(default)]
|
||||
pub backend_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
|
||||
@@ -1,56 +1,5 @@
|
||||
use super::AnalysisInput;
|
||||
|
||||
/// System prompt for the consolidation LLM.
|
||||
/// Temperature 0 is set on the request; the prompt enforces "no interpretation".
|
||||
///
|
||||
/// Sandbox-validated against case c414cf52 (3 runs each, fair pipeline with
|
||||
/// pre- and post-LLM gazetteer pass against the current vocab):
|
||||
/// - eliminates two hallucination classes the previous prompt produced —
|
||||
/// unmarked "Hypotonie" when the dictation said "Hypertonie" (0/3 vs 2/3),
|
||||
/// and inventing a unit ("35 ng/l") for a dictated value without unit
|
||||
/// (0/3 vs 2/3)
|
||||
/// - keeps Latin terms intact ("Punctum Maximum", "Apex Cordis", "Axilla"
|
||||
/// in 3/3 vs 2/3)
|
||||
/// - relies on the gazetteer for drug-name correction (Enoxaparin etc.) —
|
||||
/// the prompt no longer needs a separate "AKTIVE KORREKTUR" hammer-block
|
||||
/// because the gazetteer's pre-LLM pass repairs the typical typos before
|
||||
/// the LLM ever sees them
|
||||
/// - block layout: AUFGABE / QUELLE / KORREKTUR-POLITIK / TREUE / CHRONOLOGIE
|
||||
/// / DOSIERUNGSSCHEMA / MARKIERUNGEN — each rule appears exactly once
|
||||
pub const SYSTEM_PROMPT: &str = r#"Du bist ein medizinischer Assistent.
|
||||
|
||||
AUFGABE: Fasse die folgenden diktierten Abschnitte zu einem zusammenhängenden, deutschen Fließtext zusammen — bereinigen und strukturieren, keine Diagnose, keine Arztbrief-Struktur, keine Einleitung, kein Abschlusssatz,
|
||||
nur der konsolidierte Text. Mehrere Absätze erlaubt. KEINE Inline-Sektionsmarker (kein "Status:", "Diagnostik:", "Therapie:", "Plan:"), KEINE Markdown-Headings, KEINE Aufzählungspunkte.
|
||||
|
||||
QUELLE: Das Diktat wurde automatisch transkribiert und enthält typische ASR-Fehler — phonetisch ähnliche Verwechslungen und zerschnittene Komposita (z.B. "Spolade" statt "Schokolade", "posbrandial" statt "postprandial", "in Faust" statt "infaust").
|
||||
|
||||
MARKIERUNGEN: ==Originaltext [<Hinweistext>]== signalisiert dem Arzt "bitte prüfen". Der <Hinweistext> ist optional und beschreibt kurz, warum markiert wurde. Keine anderen Annotations-Formen ("(unsicher)", "[TODO]" o.ä.) verwenden.
|
||||
|
||||
TREUE ZUM DIKTAT (höchste Priorität — vor allen anderen Regeln):
|
||||
- Diktat-Reihenfolge erhalten.
|
||||
- Widersprüchliche Aussagen: Originaltext markieren. Nicht umformulieren, nichts ergänzen!
|
||||
- Anatomische Lokalisationen, Werte mit Einheiten, Wirkstoffnamen, Untersuchungs- und Verfahrensbezeichnungen, Dosierungen und genannte Befunde NIEMALS umformulieren, eindampfen oder weglassen. Wörtlich übernehmen!
|
||||
- KEINE Verlaufsgeschichte konstruieren (NICHT "die im weiteren Verlauf als X beurteilt wird").
|
||||
- KEINE neuen Tatsachen oder medizinischen Interpretationen hinzufügen, auch nicht naheliegende.
|
||||
- Werte mit Einheiten zusammenhalten ("35 ng/l", nicht nur "35"). Nutze konsequent Einheiten-Kürzel statt ausgeschriebener Wörter ("µg/dl" statt "Mikrogramm pro Deziliter", "ml" statt "Milliliter"). Fehlt die Einheit im Diktat: Zahl markieren.
|
||||
- Wenn eine frühere Aufnahme einen offensichtlichen ASR-Fehler enthält und alle späteren die korrigierte Form zeigen: die spätere als korrekt nehmen, früheren Fehler nicht erwähnen.
|
||||
- KRITISCH: Jegliche Ausgabe, die nicht Teil des Transkripts ist, MUSS IN "[...]" GESETZT UND MARKIERT WERDEN.
|
||||
|
||||
KORREKTUR-POLITIK:
|
||||
- Niemals raten! Bei Unsicherheit: Original behalten und markieren. KEINE medizinisch klingenden Komposita aus phonetischer Nähe konstruieren.
|
||||
- Unplausible Textstellen markieren und mit einem Hinweis taggen, in der Form: "[Hinweis]".
|
||||
|
||||
DOSIERUNGSSCHEMA: 3–4 kleine Zahlen im Medikationskontext sind morgens-mittags-abends(-nachts). "Aspirin 100mg 101" -> "Aspirin 100mg 1-0-1". Nur bei klarem Medikationskontext anwenden und bei Unsicherheit mit ==...== markieren.
|
||||
|
||||
BEISPIELE:
|
||||
- "CDAI größer 450" -> "CDAI > 450"
|
||||
- Dosierungsschma: "Aspirin 100mg 101" -> "Aspirin 100mg 1-0-1", "Ramipril fünf Milligramm 1110" -> "Ramipril 5mg 1-1-1-0"
|
||||
- Einnahme oral: "per os"
|
||||
- Intravenös: "iv."
|
||||
- Einheiten: "Mikorgramm" -> "µg"
|
||||
- „Römisch 4“ -> "VI"
|
||||
"#;
|
||||
|
||||
/// Render the user-content portion of the chat request from the persisted
|
||||
/// JSON input. Recordings with blank text are skipped — they carry no signal
|
||||
/// for the LLM and only add noise. Chronological order is taken as given;
|
||||
@@ -89,6 +38,7 @@ mod tests {
|
||||
text: text.into(),
|
||||
})
|
||||
.collect(),
|
||||
backend_id: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,16 +1,15 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
use super::backend::{default_backend, find_backend};
|
||||
use super::{
|
||||
ANALYSIS_INPUT_FILE, AnalysisInput, AnalyzeReceiver, DOCUMENT_FILE, auto_trigger, llm, prompt,
|
||||
};
|
||||
use crate::events::{self, CaseEventKind, EventSender};
|
||||
use crate::gazetteer::Gazetteer;
|
||||
use crate::settings::Settings;
|
||||
use crate::{BusyGuard, WorkerBusy};
|
||||
|
||||
/// Consume analyze jobs sequentially. The external LLM tolerates parallelism,
|
||||
@@ -18,26 +17,16 @@ use crate::{BusyGuard, WorkerBusy};
|
||||
/// provider from bursts. Trivial to lift later via `tokio::spawn(process(..))`.
|
||||
pub async fn run(
|
||||
mut rx: AnalyzeReceiver,
|
||||
settings: Arc<Settings>,
|
||||
client: reqwest::Client,
|
||||
worker_busy: WorkerBusy,
|
||||
vocab: Arc<Gazetteer>,
|
||||
events_tx: EventSender,
|
||||
) {
|
||||
info!(vocab_entries = vocab.len(), "Analyze worker started");
|
||||
let timeout = Duration::from_secs(settings.llm.timeout_seconds);
|
||||
|
||||
while let Some(job) = rx.recv().await {
|
||||
let _guard = BusyGuard::new(worker_busy.clone());
|
||||
process(
|
||||
&job.case_dir,
|
||||
&settings,
|
||||
&client,
|
||||
&vocab,
|
||||
timeout,
|
||||
&events_tx,
|
||||
)
|
||||
.await;
|
||||
process(&job.case_dir, &client, &vocab, &events_tx).await;
|
||||
}
|
||||
|
||||
warn!("Analyze worker stopped (channel closed)");
|
||||
@@ -45,10 +34,8 @@ pub async fn run(
|
||||
|
||||
async fn process(
|
||||
case_dir: &Path,
|
||||
settings: &Settings,
|
||||
client: &reqwest::Client,
|
||||
vocab: &Gazetteer,
|
||||
timeout: Duration,
|
||||
events_tx: &EventSender,
|
||||
) {
|
||||
let input_path = case_dir.join(ANALYSIS_INPUT_FILE);
|
||||
@@ -83,6 +70,7 @@ async fn process(
|
||||
case_dir,
|
||||
&input.last_recording_mtime,
|
||||
&format!("stub write: {e}"),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
@@ -103,10 +91,13 @@ async fn process(
|
||||
|
||||
let user_content = prompt::render_prompt(&input);
|
||||
let total_chars = user_content.chars().count();
|
||||
let backend = find_backend(&input.backend_id).unwrap_or_else(default_backend);
|
||||
info!(
|
||||
case = %case_dir.display(),
|
||||
recording_count = input.recordings.len(),
|
||||
total_chars,
|
||||
backend_id = %backend.id,
|
||||
model = %backend.model_id,
|
||||
"sending to llm"
|
||||
);
|
||||
|
||||
@@ -114,30 +105,17 @@ async fn process(
|
||||
// document write below, the recovery scan will re-enqueue this job and
|
||||
// the call will be made again. Accepted cost — rare event, small
|
||||
// per-call price. Response-caching in a `.tmp` file would prevent it.
|
||||
let llm_settings = llm::LlmSettings {
|
||||
base_url: &settings.llm.url,
|
||||
api_key: &settings.llm.api_key,
|
||||
model: &settings.llm.model,
|
||||
temperature: settings.llm.temperature,
|
||||
};
|
||||
let document = match llm::chat_once(
|
||||
client,
|
||||
&llm_settings,
|
||||
&settings.llm.system_prompt,
|
||||
&user_content,
|
||||
timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
let document = match llm::chat_once(client, backend, &user_content).await {
|
||||
Ok(text) => text,
|
||||
Err(e) => {
|
||||
// Do not log the response body — LlmError::Display is already
|
||||
// redacted, but reinforce the rule here for future readers.
|
||||
error!(case = %case_dir.display(), error = %e, "llm call failed");
|
||||
error!(case = %case_dir.display(), backend_id = %backend.id, error = %e, "llm call failed");
|
||||
let _ = auto_trigger::write_failure_marker(
|
||||
case_dir,
|
||||
&input.last_recording_mtime,
|
||||
&format!("llm: {e}"),
|
||||
Some(&backend.id),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
@@ -155,6 +133,7 @@ async fn process(
|
||||
case_dir,
|
||||
&input.last_recording_mtime,
|
||||
&format!("write document: {e}"),
|
||||
Some(&backend.id),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
|
||||
Reference in New Issue
Block a user