AILang/experiments/2026-05-12-cross-model-authoring/harness/src/scoring.rs

//! `scores.csv` + `summary.md` emitters.
//!
//! Columns (parent spec §Scoring):
//!   cohort, task_id, first_attempt_green, turns_to_green,
//!   prompt_tokens, completion_tokens, error_classes, final_status

use anyhow::{Context, Result};
use serde::Serialize;
use std::collections::BTreeSet;
use std::path::Path;

#[derive(Debug, Clone, Serialize)]
pub struct ScoreRow {
    pub cohort: String,
    pub task_id: String,
    pub first_attempt_green: bool,
    /// `None` means INF (never reached green).
    pub turns_to_green: Option<u32>,
    pub prompt_tokens: u64,
    pub completion_tokens: u64,
    pub error_classes: BTreeSet<String>,
    pub final_status: FinalStatus,
}

#[derive(Debug, Clone, Copy, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum FinalStatus {
    Green,
    TurnLimit,
    BudgetAbort,
    ApiFailure,
}

impl FinalStatus {
    fn as_csv(self) -> &'static str {
        match self {
            FinalStatus::Green => "green",
            FinalStatus::TurnLimit => "turn_limit",
            FinalStatus::BudgetAbort => "budget_abort",
            FinalStatus::ApiFailure => "api_failure",
        }
    }
}

pub fn write_scores_csv(rows: &[ScoreRow], path: &Path) -> Result<()> {
    use std::io::Write;
    let mut f = std::fs::File::create(path)
        .with_context(|| format!("creating {}", path.display()))?;
    writeln!(f, "cohort,task_id,first_attempt_green,turns_to_green,prompt_tokens,completion_tokens,error_classes,final_status")?;
    for r in rows {
        let turns = match r.turns_to_green {
            Some(n) => n.to_string(),
            None => "INF".to_string(),
        };
        let errs = r.error_classes.iter().cloned().collect::<Vec<_>>().join(";");
        writeln!(
            f,
            "{},{},{},{},{},{},{},{}",
            r.cohort, r.task_id, r.first_attempt_green, turns, r.prompt_tokens,
            r.completion_tokens, errs, r.final_status.as_csv()
        )?;
    }
    Ok(())
}

pub fn write_summary_md(rows: &[ScoreRow], path: &Path) -> Result<()> {
    use std::io::Write;
    let mut f = std::fs::File::create(path)
        .with_context(|| format!("creating {}", path.display()))?;
    writeln!(f, "# Cross-model authoring-form test — run summary\n")?;
    for cohort in ["json", "ail"] {
        writeln!(f, "## Cohort: {cohort}\n")?;
        let cohort_rows: Vec<&ScoreRow> = rows.iter().filter(|r| r.cohort == cohort).collect();
        if cohort_rows.is_empty() {
            writeln!(f, "_(no rows)_\n")?;
            continue;
        }
        let n_total = cohort_rows.len();
        let n_green = cohort_rows.iter().filter(|r| matches!(r.final_status, FinalStatus::Green)).count();
        let n_first = cohort_rows.iter().filter(|r| r.first_attempt_green).count();
        let mean_turns: f64 = {
            let xs: Vec<f64> = cohort_rows.iter().filter_map(|r| r.turns_to_green.map(|n| n as f64)).collect();
            if xs.is_empty() { f64::NAN } else { xs.iter().sum::<f64>() / xs.len() as f64 }
        };
        let total_prompt: u64 = cohort_rows.iter().map(|r| r.prompt_tokens).sum();
        let total_completion: u64 = cohort_rows.iter().map(|r| r.completion_tokens).sum();
        let mut error_freq: std::collections::BTreeMap<String, u32> = Default::default();
        for r in &cohort_rows {
            for e in &r.error_classes { *error_freq.entry(e.clone()).or_default() += 1; }
        }
        let top_err = error_freq.iter().max_by_key(|(_, n)| **n).map(|(k, n)| format!("{} (x{})", k, n)).unwrap_or_else(|| "(none)".to_string());
        writeln!(f, "- tasks: {n_total}")?;
        writeln!(f, "- reached green: {n_green}")?;
        writeln!(f, "- first-attempt green: {n_first}")?;
        writeln!(f, "- mean turns-to-green (green-only): {mean_turns:.2}")?;
        writeln!(f, "- total prompt tokens: {total_prompt}")?;
        writeln!(f, "- total completion tokens: {total_completion}")?;
        writeln!(f, "- most common error class: {top_err}\n")?;
    }
    Ok(())
}