From 012e6732b388f69a2d4a38f03129f9db2a611616 Mon Sep 17 00:00:00 2001 From: Brummel Date: Mon, 4 May 2026 09:17:08 +0200 Subject: [PATCH] Refactor LLM interaction to use backend catalog Introduces a new `LlmBackend` struct to centralize LLM configuration. This change refactors the `call_llm` function in both `run_full_case.rs` and `run_llm_only.rs` to accept an `LlmBackend` instance instead of individual settings. The `doctate_server::analyze::backend` module is now used to obtain the default backend, simplifying configuration and improving consistency. The `clone_backend_with_system_prompt` helper function is introduced to allow temporary modification of LLM backends for experimentation without altering the production catalog. This change improves code organization and makes it easier to manage LLM configurations across different parts of the application. --- experiments/src/bin/run_full_case.rs | 49 +++++++++++++++------------- experiments/src/bin/run_llm_only.rs | 42 +++++++++++------------- experiments/src/lib.rs | 35 +++++++++++++++++--- 3 files changed, 75 insertions(+), 51 deletions(-) diff --git a/experiments/src/bin/run_full_case.rs b/experiments/src/bin/run_full_case.rs index 93ea586..184f8eb 100644 --- a/experiments/src/bin/run_full_case.rs +++ b/experiments/src/bin/run_full_case.rs @@ -21,9 +21,11 @@ use std::time::{Duration, Instant}; use anyhow::{Context, Result}; use clap::Parser; use doctate_experiments::{ - GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write, load_prompt, run_id, + GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write, + clone_backend_with_system_prompt, load_prompt, run_id, }; -use doctate_server::analyze::llm::{self as srv_llm, LlmSettings as LlmCallSettings}; +use doctate_server::analyze::backend::{self, LlmBackend}; +use doctate_server::analyze::llm as srv_llm; use doctate_server::analyze::prompt::render_prompt; use doctate_server::analyze::{AnalysisInput, RecordingInput}; use doctate_server::gazetteer::{Gazetteer, SpellbookDict}; @@ -79,11 +81,18 @@ async fn main() -> Result<()> { let settings_path = args.settings.to_string_lossy(); let settings = Settings::load_or_default(&settings_path); - if !settings.llm_configured() && !args.no_llm { - anyhow::bail!( - "LLM not configured in {settings_path}. Set [llm].url and [llm].model, or pass --no-llm." - ); - } + let base_backend = if args.no_llm { + None + } else { + let b = backend::default_backend(); + if !b.is_available() { + anyhow::bail!( + "default LLM backend '{}' is not available — set the provider's API key env var, or pass --no-llm.", + b.id + ); + } + Some(b) + }; let llm_system_prompt = load_prompt(&args.llm_prompt)?; @@ -150,19 +159,19 @@ async fn main() -> Result<()> { let last_recording_mtime = audios .last() - .map(|p| stem(p)) + .map(stem) .unwrap_or_else(|| "unknown".to_string()); let analysis_input = AnalysisInput { last_recording_mtime, recordings: analysis_recordings, + backend_id: String::new(), }; let user_content = render_prompt(&analysis_input); atomic_write(&run_dir.join("llm_input.txt"), &user_content)?; - if !args.no_llm { + if let Some(b) = base_backend { let t_llm = Instant::now(); - let document_raw = - call_llm(&http, &settings, &llm_system_prompt, &user_content).await?; + let document_raw = call_llm(&http, b, &llm_system_prompt, &user_content).await?; timings.llm_ms = t_llm.elapsed().as_millis() as u64; // Post-LLM gazetteer pass — mirrors `server/src/analyze/worker.rs:150`. // Catches LLM-drift back to non-canonical forms (e.g. anglicization) @@ -185,9 +194,9 @@ async fn main() -> Result<()> { .unwrap_or_default(), llm_variant: llm_variant_id.clone(), whisper_url: settings.whisper.url.clone(), - llm_url: settings.llm.url.clone(), - llm_model: settings.llm.model.clone(), - llm_temperature: settings.llm.temperature, + llm_url: base_backend.map(|b| b.url.clone()).unwrap_or_default(), + llm_model: base_backend.map(|b| b.model_id.clone()).unwrap_or_default(), + llm_temperature: base_backend.map(|b| b.temperature).unwrap_or(0.0), vocab_dir: args.vocab_dir.display().to_string(), vocab_loaded: gazetteer.len(), gazetteer_dict: dict_mode_clone(&dict_mode), @@ -315,18 +324,12 @@ async fn transcribe_one( async fn call_llm( http: &reqwest::Client, - settings: &Settings, + base: &LlmBackend, system_prompt: &str, user_content: &str, ) -> Result { - let call = LlmCallSettings { - base_url: &settings.llm.url, - api_key: &settings.llm.api_key, - model: &settings.llm.model, - temperature: settings.llm.temperature, - }; - let timeout = Duration::from_secs(settings.llm.timeout_seconds); - let out = srv_llm::chat_once(http, &call, system_prompt, user_content, timeout) + let custom = clone_backend_with_system_prompt(base, system_prompt); + let out = srv_llm::chat_once(http, &custom, user_content) .await .map_err(|e| anyhow::anyhow!("llm chat failed: {e}"))?; Ok(out) diff --git a/experiments/src/bin/run_llm_only.rs b/experiments/src/bin/run_llm_only.rs index 022dd56..613e26d 100644 --- a/experiments/src/bin/run_llm_only.rs +++ b/experiments/src/bin/run_llm_only.rs @@ -23,18 +23,19 @@ fn data_runs_root(case_dir: &Path) -> anyhow::Result { std::fs::create_dir_all(&runs)?; Ok(runs) } -use std::time::{Duration, Instant}; +use std::time::Instant; use anyhow::{Context, Result}; use clap::Parser; use doctate_experiments::{ - GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write, load_prompt, run_id, + GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write, + clone_backend_with_system_prompt, load_prompt, run_id, }; -use doctate_server::analyze::llm::{self as srv_llm, LlmSettings as LlmCallSettings}; +use doctate_server::analyze::backend::{self, LlmBackend}; +use doctate_server::analyze::llm as srv_llm; use doctate_server::analyze::prompt::render_prompt; use doctate_server::analyze::{AnalysisInput, RecordingInput}; use doctate_server::gazetteer::Gazetteer; -use doctate_server::settings::Settings; use serde::Deserialize; use tracing::info; @@ -55,9 +56,6 @@ struct Args { #[arg(long, default_value_t = 1)] runs: u32, - #[arg(long, default_value = "../server/settings.toml")] - settings: PathBuf, - /// Vocab dir for the Post-LLM gazetteer pass. Mirrors the server's /// `analyze/worker.rs:150` post-AI normalization. Pass `none` to skip. #[arg(long, default_value = "../server/vocab")] @@ -83,9 +81,12 @@ async fn main() -> Result<()> { .case_dir .canonicalize() .with_context(|| format!("case-dir not found: {}", args.case_dir.display()))?; - let settings = Settings::load_or_default(&args.settings.to_string_lossy()); - if !settings.llm_configured() { - anyhow::bail!("LLM not configured in {}", args.settings.display()); + let base_backend = backend::default_backend(); + if !base_backend.is_available() { + anyhow::bail!( + "default LLM backend '{}' is not available — set the provider's API key env var", + base_backend.id + ); } let llm_system_prompt = load_prompt(&args.llm_prompt)?; @@ -111,6 +112,7 @@ async fn main() -> Result<()> { .map(|r| r.recorded_at.clone()) .unwrap_or_default(), recordings, + backend_id: String::new(), }; let user_content = render_prompt(&analysis_input); @@ -127,7 +129,7 @@ async fn main() -> Result<()> { let wallclock = Instant::now(); let t_llm = Instant::now(); - let document_raw = call_llm(&http, &settings, &llm_system_prompt, &user_content).await?; + let document_raw = call_llm(&http, base_backend, &llm_system_prompt, &user_content).await?; // Post-LLM gazetteer pass — mirrors server/src/analyze/worker.rs:150 let document = gazetteer.replace(&document_raw); let timings = RunTimings { @@ -149,9 +151,9 @@ async fn main() -> Result<()> { .unwrap_or_default(), llm_variant: llm_variant_id.clone(), whisper_url: String::new(), - llm_url: settings.llm.url.clone(), - llm_model: settings.llm.model.clone(), - llm_temperature: settings.llm.temperature, + llm_url: base_backend.url.clone(), + llm_model: base_backend.model_id.clone(), + llm_temperature: base_backend.temperature, vocab_dir: format!("{} (pre+post-LLM)", args.vocab_dir.display()), vocab_loaded: gazetteer.len(), gazetteer_dict: GazetteerDictMode::NoDict, @@ -213,18 +215,12 @@ fn stem>(p: P) -> String { async fn call_llm( http: &reqwest::Client, - settings: &Settings, + base: &LlmBackend, system_prompt: &str, user_content: &str, ) -> Result { - let call = LlmCallSettings { - base_url: &settings.llm.url, - api_key: &settings.llm.api_key, - model: &settings.llm.model, - temperature: settings.llm.temperature, - }; - let timeout = Duration::from_secs(settings.llm.timeout_seconds); - srv_llm::chat_once(http, &call, system_prompt, user_content, timeout) + let custom = clone_backend_with_system_prompt(base, system_prompt); + srv_llm::chat_once(http, &custom, user_content) .await .map_err(|e| anyhow::anyhow!("llm chat failed: {e}")) } diff --git a/experiments/src/lib.rs b/experiments/src/lib.rs index 2a010e0..f9bab26 100644 --- a/experiments/src/lib.rs +++ b/experiments/src/lib.rs @@ -4,14 +4,16 @@ //! without modifying production state. All outputs land under `case_/runs/` //! and `case_/compare/` — never in `tmpdata/`. //! -//! **Security note**: The Ionos bearer key flows through this code as a -//! `String` inside `Settings.llm.api_key`. Helpers here MUST NOT persist or -//! print the key. `RunMeta` deliberately stores only the URL/model/timestamp — -//! never the key — so on-disk audit logs are safe to share. +//! **Security note**: The provider bearer key (e.g. `IONOS_API_KEY`) is read +//! once into the server's backend catalog and lives there as a `String`. +//! Helpers here MUST NOT persist or print the key. `RunMeta` deliberately +//! stores only the URL/model/timestamp — never the key — so on-disk audit +//! logs are safe to share. use std::path::Path; use anyhow::Context; +use doctate_server::analyze::backend::LlmBackend; use serde::Serialize; use time::OffsetDateTime; use time::format_description::well_known::Iso8601; @@ -24,7 +26,7 @@ pub fn run_id(llm_variant: &str, run_index: u32) -> String { let stamp = now .format(&Iso8601::DEFAULT) .unwrap_or_else(|_| "unknown".into()); - let safe_stamp = stamp.replace(':', "-").replace('.', "-"); + let safe_stamp = stamp.replace([':', '.'], "-"); format!("{safe_stamp}_l-{llm_variant}_run{run_index}") } @@ -36,6 +38,29 @@ pub fn load_prompt(path: &Path) -> anyhow::Result { Ok(content.trim_end_matches('\n').to_string()) } +/// Build a backend struct that mirrors `base` but uses `system_prompt` as +/// the first `system` message. Used by sandbox runners to A/B different +/// prompt files against the same provider profile without touching the +/// production catalog. +pub fn clone_backend_with_system_prompt(base: &LlmBackend, system_prompt: &str) -> LlmBackend { + LlmBackend { + id: base.id.clone(), + label: base.label.clone(), + url: base.url.clone(), + api_key: base.api_key.clone(), + requires_api_key: base.requires_api_key, + model_id: base.model_id.clone(), + temperature: base.temperature, + top_p: base.top_p, + max_completion_tokens: base.max_completion_tokens, + reasoning_effort: base.reasoning_effort.clone(), + use_json_schema: base.use_json_schema, + system_prompt: system_prompt.to_string(), + format_instruction: base.format_instruction.clone(), + timeout_seconds: base.timeout_seconds, + } +} + /// Atomically write content to `path` via a sibling tempfile + rename. /// Same pattern the server uses for `document.md` — ensures readers never /// observe a partial file.