Brummel/doctate
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor LLM interaction to use backend catalog

Browse Source 
Introduces a new `LlmBackend` struct to centralize LLM configuration.
This change refactors the `call_llm` function in both `run_full_case.rs`
and `run_llm_only.rs` to accept an `LlmBackend` instance instead of
individual settings.

The `doctate_server::analyze::backend` module is now used to obtain the
default backend, simplifying configuration and improving consistency.
The `clone_backend_with_system_prompt` helper function is introduced to
allow temporary modification of LLM backends for experimentation without
altering the production catalog.

This change improves code organization and makes it easier to manage LLM
configurations across different parts of the application.
This commit is contained in:
Brummel
2026-05-04 09:17:08 +02:00
parent d86e2d7843
commit 012e6732b3
3 changed files with 75 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/src/bin/run_full_case.rs
+26 -23
View File
@@ -21,9 +21,11 @@ use std::time::{Duration, Instant};
use anyhow::{Context, Result};
use clap::Parser;
use doctate_experiments::{
GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write, load_prompt, run_id,
GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write,
clone_backend_with_system_prompt, load_prompt, run_id,
};
use doctate_server::analyze::llm::{self as srv_llm, LlmSettings as LlmCallSettings};
use doctate_server::analyze::backend::{self, LlmBackend};
use doctate_server::analyze::llm as srv_llm;
use doctate_server::analyze::prompt::render_prompt;
use doctate_server::analyze::{AnalysisInput, RecordingInput};
use doctate_server::gazetteer::{Gazetteer, SpellbookDict};
@@ -79,11 +81,18 @@ async fn main() -> Result<()> {
let settings_path = args.settings.to_string_lossy();
let settings = Settings::load_or_default(&settings_path);
if !settings.llm_configured() && !args.no_llm {
anyhow::bail!(
"LLM not configured in {settings_path}. Set [llm].url and [llm].model, or pass --no-llm."
);
}
let base_backend = if args.no_llm {
None
} else {
let b = backend::default_backend();
if !b.is_available() {
anyhow::bail!(
"default LLM backend '{}' is not available — set the provider's API key env var, or pass --no-llm.",
b.id
);
}
Some(b)
};
let llm_system_prompt = load_prompt(&args.llm_prompt)?;
@@ -150,19 +159,19 @@ async fn main() -> Result<()> {
let last_recording_mtime = audios
.last()
.map(|p| stem(p))
.map(stem)
.unwrap_or_else(|| "unknown".to_string());
let analysis_input = AnalysisInput {
last_recording_mtime,
recordings: analysis_recordings,
backend_id: String::new(),
};
let user_content = render_prompt(&analysis_input);
atomic_write(&run_dir.join("llm_input.txt"), &user_content)?;
if !args.no_llm {
if let Some(b) = base_backend {
let t_llm = Instant::now();
let document_raw =
call_llm(&http, &settings, &llm_system_prompt, &user_content).await?;
let document_raw = call_llm(&http, b, &llm_system_prompt, &user_content).await?;
timings.llm_ms = t_llm.elapsed().as_millis() as u64;
// Post-LLM gazetteer pass — mirrors `server/src/analyze/worker.rs:150`.
// Catches LLM-drift back to non-canonical forms (e.g. anglicization)
@@ -185,9 +194,9 @@ async fn main() -> Result<()> {
.unwrap_or_default(),
llm_variant: llm_variant_id.clone(),
whisper_url: settings.whisper.url.clone(),
llm_url: settings.llm.url.clone(),
llm_model: settings.llm.model.clone(),
llm_temperature: settings.llm.temperature,
llm_url: base_backend.map(|b| b.url.clone()).unwrap_or_default(),
llm_model: base_backend.map(|b| b.model_id.clone()).unwrap_or_default(),
llm_temperature: base_backend.map(|b| b.temperature).unwrap_or(0.0),
vocab_dir: args.vocab_dir.display().to_string(),
vocab_loaded: gazetteer.len(),
gazetteer_dict: dict_mode_clone(&dict_mode),
@@ -315,18 +324,12 @@ async fn transcribe_one(
async fn call_llm(
http: &reqwest::Client,
settings: &Settings,
base: &LlmBackend,
system_prompt: &str,
user_content: &str,
) -> Result<String> {
let call = LlmCallSettings {
base_url: &settings.llm.url,
api_key: &settings.llm.api_key,
model: &settings.llm.model,
temperature: settings.llm.temperature,
};
let timeout = Duration::from_secs(settings.llm.timeout_seconds);
let out = srv_llm::chat_once(http, &call, system_prompt, user_content, timeout)
let custom = clone_backend_with_system_prompt(base, system_prompt);
let out = srv_llm::chat_once(http, &custom, user_content)
.await
.map_err(|e| anyhow::anyhow!("llm chat failed: {e}"))?;
Ok(out)
experiments/src/bin/run_llm_only.rs
+19 -23
View File
@@ -23,18 +23,19 @@ fn data_runs_root(case_dir: &Path) -> anyhow::Result<PathBuf> {
std::fs::create_dir_all(&runs)?;
Ok(runs)
}
use std::time::{Duration, Instant};
use std::time::Instant;
use anyhow::{Context, Result};
use clap::Parser;
use doctate_experiments::{
GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write, load_prompt, run_id,
GazetteerDictMode, RecordingMeta, RunMeta, RunTimings, atomic_write,
clone_backend_with_system_prompt, load_prompt, run_id,
};
use doctate_server::analyze::llm::{self as srv_llm, LlmSettings as LlmCallSettings};
use doctate_server::analyze::backend::{self, LlmBackend};
use doctate_server::analyze::llm as srv_llm;
use doctate_server::analyze::prompt::render_prompt;
use doctate_server::analyze::{AnalysisInput, RecordingInput};
use doctate_server::gazetteer::Gazetteer;
use doctate_server::settings::Settings;
use serde::Deserialize;
use tracing::info;
@@ -55,9 +56,6 @@ struct Args {
#[arg(long, default_value_t = 1)]
runs: u32,
#[arg(long, default_value = "../server/settings.toml")]
settings: PathBuf,
/// Vocab dir for the Post-LLM gazetteer pass. Mirrors the server's
/// `analyze/worker.rs:150` post-AI normalization. Pass `none` to skip.
#[arg(long, default_value = "../server/vocab")]
@@ -83,9 +81,12 @@ async fn main() -> Result<()> {
.case_dir
.canonicalize()
.with_context(|| format!("case-dir not found: {}", args.case_dir.display()))?;
let settings = Settings::load_or_default(&args.settings.to_string_lossy());
if !settings.llm_configured() {
anyhow::bail!("LLM not configured in {}", args.settings.display());
let base_backend = backend::default_backend();
if !base_backend.is_available() {
anyhow::bail!(
"default LLM backend '{}' is not available — set the provider's API key env var",
base_backend.id
);
}
let llm_system_prompt = load_prompt(&args.llm_prompt)?;
@@ -111,6 +112,7 @@ async fn main() -> Result<()> {
.map(|r| r.recorded_at.clone())
.unwrap_or_default(),
recordings,
backend_id: String::new(),
};
let user_content = render_prompt(&analysis_input);
@@ -127,7 +129,7 @@ async fn main() -> Result<()> {
let wallclock = Instant::now();
let t_llm = Instant::now();
let document_raw = call_llm(&http, &settings, &llm_system_prompt, &user_content).await?;
let document_raw = call_llm(&http, base_backend, &llm_system_prompt, &user_content).await?;
// Post-LLM gazetteer pass — mirrors server/src/analyze/worker.rs:150
let document = gazetteer.replace(&document_raw);
let timings = RunTimings {
@@ -149,9 +151,9 @@ async fn main() -> Result<()> {
.unwrap_or_default(),
llm_variant: llm_variant_id.clone(),
whisper_url: String::new(),
llm_url: settings.llm.url.clone(),
llm_model: settings.llm.model.clone(),
llm_temperature: settings.llm.temperature,
llm_url: base_backend.url.clone(),
llm_model: base_backend.model_id.clone(),
llm_temperature: base_backend.temperature,
vocab_dir: format!("{} (pre+post-LLM)", args.vocab_dir.display()),
vocab_loaded: gazetteer.len(),
gazetteer_dict: GazetteerDictMode::NoDict,
@@ -213,18 +215,12 @@ fn stem<P: AsRef<Path>>(p: P) -> String {
async fn call_llm(
http: &reqwest::Client,
settings: &Settings,
base: &LlmBackend,
system_prompt: &str,
user_content: &str,
) -> Result<String> {
let call = LlmCallSettings {
base_url: &settings.llm.url,
api_key: &settings.llm.api_key,
model: &settings.llm.model,
temperature: settings.llm.temperature,
};
let timeout = Duration::from_secs(settings.llm.timeout_seconds);
srv_llm::chat_once(http, &call, system_prompt, user_content, timeout)
let custom = clone_backend_with_system_prompt(base, system_prompt);
srv_llm::chat_once(http, &custom, user_content)
.await
.map_err(|e| anyhow::anyhow!("llm chat failed: {e}"))
}