Brummel/doctate
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix gpt-oss-120b with reasoning effort

Browse Source 
This commit adjusts the configuration for the `gpt-oss-120b` model to
run in plain-text mode. This is necessary because when
`reasoning_effort` is set to "medium", the reasoning tokens combined
with schema-constrained decoding can exhaust the `max_completion_tokens`
budget, leading to an empty content response.

The change restores the previous behavior where `gpt-oss-120b` was
configured without `response_format` or `top_p`, ensuring the request
body sent to Ionos remains byte-identical to the pre-multi-backend
state. This prevents regressions and maintains stability for existing
production workflows.
This commit is contained in:
Brummel
2026-05-03 16:43:17 +02:00
parent 6fac8775e1
commit 0848e9581f
2 changed files with 107 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/analyze/backend.rs
+25 -6
View File
@@ -95,6 +95,11 @@ impl LlmBackend {
self
}
fn with_use_json_schema(mut self, use_schema: bool) -> Self {
self.use_json_schema = use_schema;
self
}
fn with_system_prompt(mut self, prompt: &str) -> Self {
self.system_prompt = prompt.into();
self
@@ -144,8 +149,16 @@ pub fn backends() -> &'static [LlmBackend] {
static B: OnceLock<Vec<LlmBackend>> = OnceLock::new();
B.get_or_init(|| {
vec![
// gpt-oss-120b: schema OFF on purpose. With `reasoning_effort:
// medium`, reasoning tokens count against `max_completion_tokens`
// (4096); on large bodies (e.g. 6 recordings, ~3.8k chars) the
// reasoning eats the budget and the model returns
// `choices[0].message.content == null` — the LlmError::Parse path.
// Pre-multi-backend (commit bf6464d) gpt-oss ran without schema
// and worked fine; this restores that body byte-for-byte.
ionos_backend("gpt_oss_120b", "GPT OSS 120b", "openai/gpt-oss-120b")
.with_reasoning_effort("medium"),
.with_reasoning_effort("medium")
.with_use_json_schema(false),
ionos_backend(
"llama_3_1_405b",
"Llama 3.1 405B",
@@ -290,15 +303,21 @@ mod tests {
assert!(b.use_json_schema, "Llama needs schema-enforced stop");
}
/// gpt-oss-120b is schema-aware out of the box. Asserting that it has
/// neither a top_p nor a format instruction guarantees the body sent to
/// Ionos stays byte-identical to pre-Llama-fix builds — the production
/// path that has been working for weeks must not silently change.
/// gpt-oss-120b runs in plain-text mode (no `response_format`, no
/// `top_p`, no second system message). Reason: with
/// `reasoning_effort: medium`, schema-constrained decoding plus reasoning
/// can exhaust `max_completion_tokens` on large bodies and return
/// `content: null`. This pin guarantees the body sent to Ionos stays
/// byte-identical to the pre-multi-backend path (commit bf6464d).
#[test]
fn gpt_oss_has_no_format_instruction_and_no_top_p() {
fn gpt_oss_runs_in_plain_text_mode_no_schema_no_top_p_no_format_hint() {
let b = find_backend("gpt_oss_120b").unwrap();
assert!((b.temperature - 0.5).abs() < f32::EPSILON);
assert!(b.top_p.is_none(), "gpt-oss must not send top_p");
assert!(
!b.use_json_schema,
"gpt-oss must NOT use json_schema — collides with reasoning_effort budget"
);
assert!(
b.format_instruction.is_none(),
"gpt-oss must not get a second system message — body byte-stable"