Refactor LLM interaction to use backend catalog

Introduces a new `LlmBackend` struct to centralize LLM configuration.
This change refactors the `call_llm` function in both `run_full_case.rs`
and `run_llm_only.rs` to accept an `LlmBackend` instance instead of
individual settings.

The `doctate_server::analyze::backend` module is now used to obtain the
default backend, simplifying configuration and improving consistency.
The `clone_backend_with_system_prompt` helper function is introduced to
allow temporary modification of LLM backends for experimentation without
altering the production catalog.

This change improves code organization and makes it easier to manage LLM
configurations across different parts of the application.

experiments/src/bin/run_full_case.rs

@@ -21,9 +21,11 @@ use std::time::{Duration, Instant};
 use anyhow::{Context, Result};
 use clap::Parser;
 use doctate_experiments::{
-    GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write, load_prompt, run_id,
+    GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write,
+    clone_backend_with_system_prompt, load_prompt, run_id,
 };
-use doctate_server::analyze::llm::{self as srv_llm, LlmSettings as LlmCallSettings};
+use doctate_server::analyze::backend::{self, LlmBackend};
+use doctate_server::analyze::llm as srv_llm;
 use doctate_server::analyze::prompt::render_prompt;
 use doctate_server::analyze::{AnalysisInput, RecordingInput};
 use doctate_server::gazetteer::{Gazetteer, SpellbookDict};
@@ -79,11 +81,18 @@ async fn main() -> Result<()> {
 
     let settings_path = args.settings.to_string_lossy();
     let settings = Settings::load_or_default(&settings_path);
-    if !settings.llm_configured() && !args.no_llm {
+    let base_backend = if args.no_llm {
+        None
+    } else {
+        let b = backend::default_backend();
+        if !b.is_available() {
             anyhow::bail!(
-            "LLM not configured in {settings_path}. Set [llm].url and [llm].model, or pass --no-llm."
+                "default LLM backend '{}' is not available — set the provider's API key env var, or pass --no-llm.",
+                b.id
             );
         }
+        Some(b)
+    };
 
     let llm_system_prompt = load_prompt(&args.llm_prompt)?;
 
@@ -150,19 +159,19 @@ async fn main() -> Result<()> {
 
         let last_recording_mtime = audios
             .last()
-            .map(|p| stem(p))
+            .map(stem)
             .unwrap_or_else(|| "unknown".to_string());
         let analysis_input = AnalysisInput {
             last_recording_mtime,
             recordings: analysis_recordings,
+            backend_id: String::new(),
         };
         let user_content = render_prompt(&analysis_input);
         atomic_write(&run_dir.join("llm_input.txt"), &user_content)?;
 
-        if !args.no_llm {
+        if let Some(b) = base_backend {
             let t_llm = Instant::now();
-            let document_raw =
+            let document_raw = call_llm(&http, b, &llm_system_prompt, &user_content).await?;
-                call_llm(&http, &settings, &llm_system_prompt, &user_content).await?;
             timings.llm_ms = t_llm.elapsed().as_millis() as u64;
             // Post-LLM gazetteer pass — mirrors `server/src/analyze/worker.rs:150`.
             // Catches LLM-drift back to non-canonical forms (e.g. anglicization)
@@ -185,9 +194,9 @@ async fn main() -> Result<()> {
                 .unwrap_or_default(),
             llm_variant: llm_variant_id.clone(),
             whisper_url: settings.whisper.url.clone(),
-            llm_url: settings.llm.url.clone(),
+            llm_url: base_backend.map(|b| b.url.clone()).unwrap_or_default(),
-            llm_model: settings.llm.model.clone(),
+            llm_model: base_backend.map(|b| b.model_id.clone()).unwrap_or_default(),
-            llm_temperature: settings.llm.temperature,
+            llm_temperature: base_backend.map(|b| b.temperature).unwrap_or(0.0),
             vocab_dir: args.vocab_dir.display().to_string(),
             vocab_loaded: gazetteer.len(),
             gazetteer_dict: dict_mode_clone(&dict_mode),
@@ -315,18 +324,12 @@ async fn transcribe_one(
 
 async fn call_llm(
     http: &reqwest::Client,
-    settings: &Settings,
+    base: &LlmBackend,
     system_prompt: &str,
     user_content: &str,
 ) -> Result<String> {
-    let call = LlmCallSettings {
+    let custom = clone_backend_with_system_prompt(base, system_prompt);
-        base_url: &settings.llm.url,
+    let out = srv_llm::chat_once(http, &custom, user_content)
-        api_key: &settings.llm.api_key,
-        model: &settings.llm.model,
-        temperature: settings.llm.temperature,
-    };
-    let timeout = Duration::from_secs(settings.llm.timeout_seconds);
-    let out = srv_llm::chat_once(http, &call, system_prompt, user_content, timeout)
         .await
         .map_err(|e| anyhow::anyhow!("llm chat failed: {e}"))?;
     Ok(out)

experiments/src/bin/run_llm_only.rs

@@ -23,18 +23,19 @@ fn data_runs_root(case_dir: &Path) -> anyhow::Result<PathBuf> {
     std::fs::create_dir_all(&runs)?;
     Ok(runs)
 }
-use std::time::{Duration, Instant};
+use std::time::Instant;
 
 use anyhow::{Context, Result};
 use clap::Parser;
 use doctate_experiments::{
-    GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write, load_prompt, run_id,
+    GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write,
+    clone_backend_with_system_prompt, load_prompt, run_id,
 };
-use doctate_server::analyze::llm::{self as srv_llm, LlmSettings as LlmCallSettings};
+use doctate_server::analyze::backend::{self, LlmBackend};
+use doctate_server::analyze::llm as srv_llm;
 use doctate_server::analyze::prompt::render_prompt;
 use doctate_server::analyze::{AnalysisInput, RecordingInput};
 use doctate_server::gazetteer::Gazetteer;
-use doctate_server::settings::Settings;
 use serde::Deserialize;
 use tracing::info;
 
@@ -55,9 +56,6 @@ struct Args {
     #[arg(long, default_value_t = 1)]
     runs: u32,
 
-    #[arg(long, default_value = "../server/settings.toml")]
-    settings: PathBuf,
-
     /// Vocab dir for the Post-LLM gazetteer pass. Mirrors the server's
     /// `analyze/worker.rs:150` post-AI normalization. Pass `none` to skip.
     #[arg(long, default_value = "../server/vocab")]
@@ -83,9 +81,12 @@ async fn main() -> Result<()> {
         .case_dir
         .canonicalize()
         .with_context(|| format!("case-dir not found: {}", args.case_dir.display()))?;
-    let settings = Settings::load_or_default(&args.settings.to_string_lossy());
+    let base_backend = backend::default_backend();
-    if !settings.llm_configured() {
+    if !base_backend.is_available() {
-        anyhow::bail!("LLM not configured in {}", args.settings.display());
+        anyhow::bail!(
+            "default LLM backend '{}' is not available — set the provider's API key env var",
+            base_backend.id
+        );
     }
 
     let llm_system_prompt = load_prompt(&args.llm_prompt)?;
@@ -111,6 +112,7 @@ async fn main() -> Result<()> {
             .map(|r| r.recorded_at.clone())
             .unwrap_or_default(),
         recordings,
+        backend_id: String::new(),
     };
     let user_content = render_prompt(&analysis_input);
 
@@ -127,7 +129,7 @@ async fn main() -> Result<()> {
 
         let wallclock = Instant::now();
         let t_llm = Instant::now();
-        let document_raw = call_llm(&http, &settings, &llm_system_prompt, &user_content).await?;
+        let document_raw = call_llm(&http, base_backend, &llm_system_prompt, &user_content).await?;
         // Post-LLM gazetteer pass — mirrors server/src/analyze/worker.rs:150
         let document = gazetteer.replace(&document_raw);
         let timings = RunTimings {
@@ -149,9 +151,9 @@ async fn main() -> Result<()> {
                 .unwrap_or_default(),
             llm_variant: llm_variant_id.clone(),
             whisper_url: String::new(),
-            llm_url: settings.llm.url.clone(),
+            llm_url: base_backend.url.clone(),
-            llm_model: settings.llm.model.clone(),
+            llm_model: base_backend.model_id.clone(),
-            llm_temperature: settings.llm.temperature,
+            llm_temperature: base_backend.temperature,
             vocab_dir: format!("{} (pre+post-LLM)", args.vocab_dir.display()),
             vocab_loaded: gazetteer.len(),
             gazetteer_dict: GazetteerDictMode::NoDict,
@@ -213,18 +215,12 @@ fn stem<P: AsRef<Path>>(p: P) -> String {
 
 async fn call_llm(
     http: &reqwest::Client,
-    settings: &Settings,
+    base: &LlmBackend,
     system_prompt: &str,
     user_content: &str,
 ) -> Result<String> {
-    let call = LlmCallSettings {
+    let custom = clone_backend_with_system_prompt(base, system_prompt);
-        base_url: &settings.llm.url,
+    srv_llm::chat_once(http, &custom, user_content)
-        api_key: &settings.llm.api_key,
-        model: &settings.llm.model,
-        temperature: settings.llm.temperature,
-    };
-    let timeout = Duration::from_secs(settings.llm.timeout_seconds);
-    srv_llm::chat_once(http, &call, system_prompt, user_content, timeout)
         .await
         .map_err(|e| anyhow::anyhow!("llm chat failed: {e}"))
 }

experiments/src/lib.rs

@@ -4,14 +4,16 @@
 //! without modifying production state. All outputs land under `case_<id>/runs/`
 //! and `case_<id>/compare/` — never in `tmpdata/`.
 //!
-//! **Security note**: The Ionos bearer key flows through this code as a
+//! **Security note**: The provider bearer key (e.g. `IONOS_API_KEY`) is read
-//! `String` inside `Settings.llm.api_key`. Helpers here MUST NOT persist or
+//! once into the server's backend catalog and lives there as a `String`.
-//! print the key. `RunMeta` deliberately stores only the URL/model/timestamp —
+//! Helpers here MUST NOT persist or print the key. `RunMeta` deliberately
-//! never the key — so on-disk audit logs are safe to share.
+//! stores only the URL/model/timestamp — never the key — so on-disk audit
+//! logs are safe to share.
 
 use std::path::Path;
 
 use anyhow::Context;
+use doctate_server::analyze::backend::LlmBackend;
 use serde::Serialize;
 use time::OffsetDateTime;
 use time::format_description::well_known::Iso8601;
@@ -24,7 +26,7 @@ pub fn run_id(llm_variant: &str, run_index: u32) -> String {
     let stamp = now
         .format(&Iso8601::DEFAULT)
         .unwrap_or_else(|_| "unknown".into());
-    let safe_stamp = stamp.replace(':', "-").replace('.', "-");
+    let safe_stamp = stamp.replace([':', '.'], "-");
     format!("{safe_stamp}_l-{llm_variant}_run{run_index}")
 }
 
@@ -36,6 +38,29 @@ pub fn load_prompt(path: &Path) -> anyhow::Result<String> {
     Ok(content.trim_end_matches('\n').to_string())
 }
 
+/// Build a backend struct that mirrors `base` but uses `system_prompt` as
+/// the first `system` message. Used by sandbox runners to A/B different
+/// prompt files against the same provider profile without touching the
+/// production catalog.
+pub fn clone_backend_with_system_prompt(base: &LlmBackend, system_prompt: &str) -> LlmBackend {
+    LlmBackend {
+        id: base.id.clone(),
+        label: base.label.clone(),
+        url: base.url.clone(),
+        api_key: base.api_key.clone(),
+        requires_api_key: base.requires_api_key,
+        model_id: base.model_id.clone(),
+        temperature: base.temperature,
+        top_p: base.top_p,
+        max_completion_tokens: base.max_completion_tokens,
+        reasoning_effort: base.reasoning_effort.clone(),
+        use_json_schema: base.use_json_schema,
+        system_prompt: system_prompt.to_string(),
+        format_instruction: base.format_instruction.clone(),
+        timeout_seconds: base.timeout_seconds,
+    }
+}
+
 /// Atomically write content to `path` via a sibling tempfile + rename.
 /// Same pattern the server uses for `document.md` — ensures readers never
 /// observe a partial file.