feat(analyze): structured LLM failure diagnostics + SSE reload on failure
Failure markers now carry finish_reason and usage tokens (prompt / completion / reasoning) pulled from the chat-completions response, so the case-page "Technische Details" block can show why an analysis truncated — e.g. gpt-oss-120b hitting max_completion_tokens with the chain-of-thought before producing any visible answer. The worker also emits a new CaseEventKind::AnalysisFailed after writing the marker, so subscribed browsers reload immediately instead of leaving the page on the "Wird analysiert …" placeholder.
This commit is contained in:
@@ -0,0 +1,162 @@
|
||||
//! Regression test for the diagnostic surface around an empty-content
|
||||
//! LLM response.
|
||||
//!
|
||||
//! Pre-fix behaviour: when the provider returned `200 OK` with
|
||||
//! `choices[0].message.content == null` (typical for reasoning models that
|
||||
//! exhausted `max_completion_tokens` on the chain-of-thought), the worker
|
||||
//! wrote a failure marker that read `llm: missing choices[0].message.content`
|
||||
//! — true but useless. Diagnostic fields the provider sent in the same
|
||||
//! response (`finish_reason`, `usage.completion_tokens`, reasoning_tokens)
|
||||
//! were dropped on the floor.
|
||||
//!
|
||||
//! This test pins the new `LlmError::EmptyContent` variant to the parsed
|
||||
//! response fields so the failure-marker UI can render structured details.
|
||||
|
||||
use doctate_server::analyze::backend::LlmBackend;
|
||||
use doctate_server::analyze::llm::{LlmError, chat_once};
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
fn test_backend(uri: String) -> LlmBackend {
|
||||
LlmBackend {
|
||||
id: "test_backend".into(),
|
||||
label: "Test".into(),
|
||||
url: uri,
|
||||
api_key: "test-key".into(),
|
||||
requires_api_key: true,
|
||||
model_id: "openai/gpt-oss-120b".into(),
|
||||
temperature: 0.2,
|
||||
top_p: None,
|
||||
max_completion_tokens: 16384,
|
||||
reasoning_effort: Some("high".into()),
|
||||
use_json_schema: false,
|
||||
system_prompt: "system".into(),
|
||||
format_instruction: None,
|
||||
timeout_seconds: 30,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_content_with_full_usage_surfaces_finish_reason_and_tokens() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||
"choices": [{
|
||||
"message": { "content": null },
|
||||
"finish_reason": "length"
|
||||
}],
|
||||
"usage": {
|
||||
"prompt_tokens": 1287,
|
||||
"completion_tokens": 16384,
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 14102
|
||||
}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let backend = test_backend(server.uri());
|
||||
let client = reqwest::Client::new();
|
||||
let err = chat_once(&client, &backend, "user content")
|
||||
.await
|
||||
.expect_err("must fail when content is null");
|
||||
|
||||
let d = err
|
||||
.empty_content_details()
|
||||
.expect("must classify as EmptyContent");
|
||||
assert_eq!(d.finish_reason, Some("length"));
|
||||
assert_eq!(d.prompt_tokens, Some(1287));
|
||||
assert_eq!(d.completion_tokens, Some(16384));
|
||||
assert_eq!(d.reasoning_tokens, Some(14102));
|
||||
assert_eq!(d.max_completion_tokens, 16384);
|
||||
|
||||
// Display string must mention finish_reason and the X/Y token ratio so
|
||||
// the failure-marker `reason` field already carries a hint even when
|
||||
// structured details are not yet rendered.
|
||||
let s = err.to_string();
|
||||
assert!(s.contains("finish_reason=length"), "got: {s}");
|
||||
assert!(s.contains("16384/16384"), "got: {s}");
|
||||
assert!(s.contains("reasoning_tokens=14102"), "got: {s}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_content_without_usage_degrades_gracefully() {
|
||||
// Some OpenAI-compatible providers omit `usage` entirely on null-content
|
||||
// responses. The parser must still classify this as EmptyContent and not
|
||||
// crash on the missing field.
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||
"choices": [{
|
||||
"message": { "content": null },
|
||||
"finish_reason": "stop"
|
||||
}]
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let backend = test_backend(server.uri());
|
||||
let client = reqwest::Client::new();
|
||||
let err = chat_once(&client, &backend, "user content")
|
||||
.await
|
||||
.expect_err("must fail when content is null");
|
||||
|
||||
let d = err
|
||||
.empty_content_details()
|
||||
.expect("must classify as EmptyContent even without usage");
|
||||
assert_eq!(d.finish_reason, Some("stop"));
|
||||
assert_eq!(d.prompt_tokens, None);
|
||||
assert_eq!(d.completion_tokens, None);
|
||||
assert_eq!(d.reasoning_tokens, None);
|
||||
assert_eq!(d.max_completion_tokens, 16384);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn populated_content_returns_ok() {
|
||||
// Sanity check: the new code path must not regress the happy path.
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||
"choices": [{
|
||||
"message": { "content": "Hallo Welt." },
|
||||
"finish_reason": "stop"
|
||||
}],
|
||||
"usage": {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let backend = test_backend(server.uri());
|
||||
let client = reqwest::Client::new();
|
||||
let text = chat_once(&client, &backend, "user content")
|
||||
.await
|
||||
.expect("must succeed when content is populated");
|
||||
assert_eq!(text, "Hallo Welt.");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_error_does_not_carry_empty_content_details() {
|
||||
// Status errors are a different LlmError variant — `empty_content_details`
|
||||
// must not falsely classify them as EmptyContent.
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(503).set_body_string("upstream down"))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let backend = test_backend(server.uri());
|
||||
let client = reqwest::Client::new();
|
||||
let err = chat_once(&client, &backend, "user content")
|
||||
.await
|
||||
.expect_err("must fail on 503");
|
||||
assert!(matches!(err, LlmError::Status { status: 503 }));
|
||||
assert!(err.empty_content_details().is_none());
|
||||
}
|
||||
Reference in New Issue
Block a user