diff --git a/experiments/src/bin/run_full_case.rs b/experiments/src/bin/run_full_case.rs index 93ea586..184f8eb 100644 --- a/experiments/src/bin/run_full_case.rs +++ b/experiments/src/bin/run_full_case.rs @@ -21,9 +21,11 @@ use std::time::{Duration, Instant}; use anyhow::{Context, Result}; use clap::Parser; use doctate_experiments::{ - GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write, load_prompt, run_id, + GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write, + clone_backend_with_system_prompt, load_prompt, run_id, }; -use doctate_server::analyze::llm::{self as srv_llm, LlmSettings as LlmCallSettings}; +use doctate_server::analyze::backend::{self, LlmBackend}; +use doctate_server::analyze::llm as srv_llm; use doctate_server::analyze::prompt::render_prompt; use doctate_server::analyze::{AnalysisInput, RecordingInput}; use doctate_server::gazetteer::{Gazetteer, SpellbookDict}; @@ -79,11 +81,18 @@ async fn main() -> Result<()> { let settings_path = args.settings.to_string_lossy(); let settings = Settings::load_or_default(&settings_path); - if !settings.llm_configured() && !args.no_llm { - anyhow::bail!( - "LLM not configured in {settings_path}. Set [llm].url and [llm].model, or pass --no-llm." - ); - } + let base_backend = if args.no_llm { + None + } else { + let b = backend::default_backend(); + if !b.is_available() { + anyhow::bail!( + "default LLM backend '{}' is not available — set the provider's API key env var, or pass --no-llm.", + b.id + ); + } + Some(b) + }; let llm_system_prompt = load_prompt(&args.llm_prompt)?; @@ -150,19 +159,19 @@ async fn main() -> Result<()> { let last_recording_mtime = audios .last() - .map(|p| stem(p)) + .map(stem) .unwrap_or_else(|| "unknown".to_string()); let analysis_input = AnalysisInput { last_recording_mtime, recordings: analysis_recordings, + backend_id: String::new(), }; let user_content = render_prompt(&analysis_input); atomic_write(&run_dir.join("llm_input.txt"), &user_content)?; - if !args.no_llm { + if let Some(b) = base_backend { let t_llm = Instant::now(); - let document_raw = - call_llm(&http, &settings, &llm_system_prompt, &user_content).await?; + let document_raw = call_llm(&http, b, &llm_system_prompt, &user_content).await?; timings.llm_ms = t_llm.elapsed().as_millis() as u64; // Post-LLM gazetteer pass — mirrors `server/src/analyze/worker.rs:150`. // Catches LLM-drift back to non-canonical forms (e.g. anglicization) @@ -185,9 +194,9 @@ async fn main() -> Result<()> { .unwrap_or_default(), llm_variant: llm_variant_id.clone(), whisper_url: settings.whisper.url.clone(), - llm_url: settings.llm.url.clone(), - llm_model: settings.llm.model.clone(), - llm_temperature: settings.llm.temperature, + llm_url: base_backend.map(|b| b.url.clone()).unwrap_or_default(), + llm_model: base_backend.map(|b| b.model_id.clone()).unwrap_or_default(), + llm_temperature: base_backend.map(|b| b.temperature).unwrap_or(0.0), vocab_dir: args.vocab_dir.display().to_string(), vocab_loaded: gazetteer.len(), gazetteer_dict: dict_mode_clone(&dict_mode), @@ -315,18 +324,12 @@ async fn transcribe_one( async fn call_llm( http: &reqwest::Client, - settings: &Settings, + base: &LlmBackend, system_prompt: &str, user_content: &str, ) -> Result { - let call = LlmCallSettings { - base_url: &settings.llm.url, - api_key: &settings.llm.api_key, - model: &settings.llm.model, - temperature: settings.llm.temperature, - }; - let timeout = Duration::from_secs(settings.llm.timeout_seconds); - let out = srv_llm::chat_once(http, &call, system_prompt, user_content, timeout) + let custom = clone_backend_with_system_prompt(base, system_prompt); + let out = srv_llm::chat_once(http, &custom, user_content) .await .map_err(|e| anyhow::anyhow!("llm chat failed: {e}"))?; Ok(out) diff --git a/experiments/src/bin/run_llm_only.rs b/experiments/src/bin/run_llm_only.rs index 022dd56..613e26d 100644 --- a/experiments/src/bin/run_llm_only.rs +++ b/experiments/src/bin/run_llm_only.rs @@ -23,18 +23,19 @@ fn data_runs_root(case_dir: &Path) -> anyhow::Result { std::fs::create_dir_all(&runs)?; Ok(runs) } -use std::time::{Duration, Instant}; +use std::time::Instant; use anyhow::{Context, Result}; use clap::Parser; use doctate_experiments::{ - GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write, load_prompt, run_id, + GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write, + clone_backend_with_system_prompt, load_prompt, run_id, }; -use doctate_server::analyze::llm::{self as srv_llm, LlmSettings as LlmCallSettings}; +use doctate_server::analyze::backend::{self, LlmBackend}; +use doctate_server::analyze::llm as srv_llm; use doctate_server::analyze::prompt::render_prompt; use doctate_server::analyze::{AnalysisInput, RecordingInput}; use doctate_server::gazetteer::Gazetteer; -use doctate_server::settings::Settings; use serde::Deserialize; use tracing::info; @@ -55,9 +56,6 @@ struct Args { #[arg(long, default_value_t = 1)] runs: u32, - #[arg(long, default_value = "../server/settings.toml")] - settings: PathBuf, - /// Vocab dir for the Post-LLM gazetteer pass. Mirrors the server's /// `analyze/worker.rs:150` post-AI normalization. Pass `none` to skip. #[arg(long, default_value = "../server/vocab")] @@ -83,9 +81,12 @@ async fn main() -> Result<()> { .case_dir .canonicalize() .with_context(|| format!("case-dir not found: {}", args.case_dir.display()))?; - let settings = Settings::load_or_default(&args.settings.to_string_lossy()); - if !settings.llm_configured() { - anyhow::bail!("LLM not configured in {}", args.settings.display()); + let base_backend = backend::default_backend(); + if !base_backend.is_available() { + anyhow::bail!( + "default LLM backend '{}' is not available — set the provider's API key env var", + base_backend.id + ); } let llm_system_prompt = load_prompt(&args.llm_prompt)?; @@ -111,6 +112,7 @@ async fn main() -> Result<()> { .map(|r| r.recorded_at.clone()) .unwrap_or_default(), recordings, + backend_id: String::new(), }; let user_content = render_prompt(&analysis_input); @@ -127,7 +129,7 @@ async fn main() -> Result<()> { let wallclock = Instant::now(); let t_llm = Instant::now(); - let document_raw = call_llm(&http, &settings, &llm_system_prompt, &user_content).await?; + let document_raw = call_llm(&http, base_backend, &llm_system_prompt, &user_content).await?; // Post-LLM gazetteer pass — mirrors server/src/analyze/worker.rs:150 let document = gazetteer.replace(&document_raw); let timings = RunTimings { @@ -149,9 +151,9 @@ async fn main() -> Result<()> { .unwrap_or_default(), llm_variant: llm_variant_id.clone(), whisper_url: String::new(), - llm_url: settings.llm.url.clone(), - llm_model: settings.llm.model.clone(), - llm_temperature: settings.llm.temperature, + llm_url: base_backend.url.clone(), + llm_model: base_backend.model_id.clone(), + llm_temperature: base_backend.temperature, vocab_dir: format!("{} (pre+post-LLM)", args.vocab_dir.display()), vocab_loaded: gazetteer.len(), gazetteer_dict: GazetteerDictMode::NoDict, @@ -213,18 +215,12 @@ fn stem>(p: P) -> String { async fn call_llm( http: &reqwest::Client, - settings: &Settings, + base: &LlmBackend, system_prompt: &str, user_content: &str, ) -> Result { - let call = LlmCallSettings { - base_url: &settings.llm.url, - api_key: &settings.llm.api_key, - model: &settings.llm.model, - temperature: settings.llm.temperature, - }; - let timeout = Duration::from_secs(settings.llm.timeout_seconds); - srv_llm::chat_once(http, &call, system_prompt, user_content, timeout) + let custom = clone_backend_with_system_prompt(base, system_prompt); + srv_llm::chat_once(http, &custom, user_content) .await .map_err(|e| anyhow::anyhow!("llm chat failed: {e}")) } diff --git a/experiments/src/lib.rs b/experiments/src/lib.rs index 2a010e0..f9bab26 100644 --- a/experiments/src/lib.rs +++ b/experiments/src/lib.rs @@ -4,14 +4,16 @@ //! without modifying production state. All outputs land under `case_/runs/` //! and `case_/compare/` — never in `tmpdata/`. //! -//! **Security note**: The Ionos bearer key flows through this code as a -//! `String` inside `Settings.llm.api_key`. Helpers here MUST NOT persist or -//! print the key. `RunMeta` deliberately stores only the URL/model/timestamp — -//! never the key — so on-disk audit logs are safe to share. +//! **Security note**: The provider bearer key (e.g. `IONOS_API_KEY`) is read +//! once into the server's backend catalog and lives there as a `String`. +//! Helpers here MUST NOT persist or print the key. `RunMeta` deliberately +//! stores only the URL/model/timestamp — never the key — so on-disk audit +//! logs are safe to share. use std::path::Path; use anyhow::Context; +use doctate_server::analyze::backend::LlmBackend; use serde::Serialize; use time::OffsetDateTime; use time::format_description::well_known::Iso8601; @@ -24,7 +26,7 @@ pub fn run_id(llm_variant: &str, run_index: u32) -> String { let stamp = now .format(&Iso8601::DEFAULT) .unwrap_or_else(|_| "unknown".into()); - let safe_stamp = stamp.replace(':', "-").replace('.', "-"); + let safe_stamp = stamp.replace([':', '.'], "-"); format!("{safe_stamp}_l-{llm_variant}_run{run_index}") } @@ -36,6 +38,29 @@ pub fn load_prompt(path: &Path) -> anyhow::Result { Ok(content.trim_end_matches('\n').to_string()) } +/// Build a backend struct that mirrors `base` but uses `system_prompt` as +/// the first `system` message. Used by sandbox runners to A/B different +/// prompt files against the same provider profile without touching the +/// production catalog. +pub fn clone_backend_with_system_prompt(base: &LlmBackend, system_prompt: &str) -> LlmBackend { + LlmBackend { + id: base.id.clone(), + label: base.label.clone(), + url: base.url.clone(), + api_key: base.api_key.clone(), + requires_api_key: base.requires_api_key, + model_id: base.model_id.clone(), + temperature: base.temperature, + top_p: base.top_p, + max_completion_tokens: base.max_completion_tokens, + reasoning_effort: base.reasoning_effort.clone(), + use_json_schema: base.use_json_schema, + system_prompt: system_prompt.to_string(), + format_instruction: base.format_instruction.clone(), + timeout_seconds: base.timeout_seconds, + } +} + /// Atomically write content to `path` via a sibling tempfile + rename. /// Same pattern the server uses for `document.md` — ensures readers never /// observe a partial file.