b0f139f389
This commit introduces a new `tester` module to the `ast` crate, enabling functional tests and performance benchmarks for MYC scripts. Functional tests are executed by reading `.myc` files from the `examples` directory, parsing expected output comments, and comparing them against the actual script execution results. Benchmarking involves measuring the execution time of scripts, calculating median durations, and comparing them against stored baselines. The commit also adds support for updating these baselines. The `ast.rs` CLI and the `main.rs` GUI application have been updated to expose these new testing and benchmarking features. The `Cargo.toml` and `Cargo.lock` files have been updated to include the `regex` dependency required for parsing benchmark comments.
127 lines
5.1 KiB
Rust
127 lines
5.1 KiB
Rust
use crate::ast::environment::Environment;
|
|
use std::fs;
|
|
use std::time::{Duration, Instant};
|
|
use regex::Regex;
|
|
|
|
pub struct TestResult {
|
|
pub name: String,
|
|
pub success: bool,
|
|
pub message: String,
|
|
}
|
|
|
|
pub struct BenchmarkResult {
|
|
pub name: String,
|
|
pub median: Duration,
|
|
pub baseline: Option<Duration>,
|
|
pub diff_pct: Option<f64>,
|
|
pub status: String,
|
|
}
|
|
|
|
pub fn run_functional_tests() -> Vec<TestResult> {
|
|
let mut results = Vec::new();
|
|
let entries = fs::read_dir("examples").unwrap();
|
|
let env = Environment::new();
|
|
|
|
for entry in entries.filter_map(|e| e.ok()) {
|
|
let path = entry.path();
|
|
if path.extension().map_or(false, |ext| ext == "myc") {
|
|
let content = fs::read_to_string(&path).unwrap();
|
|
let name = path.file_name().unwrap().to_string_lossy().to_string();
|
|
|
|
let expected_output = Regex::new(r";; Output: (.*)").unwrap()
|
|
.captures(&content)
|
|
.map(|m| m.get(1).unwrap().as_str().trim().to_string());
|
|
|
|
if let Some(expected) = expected_output {
|
|
match env.run_script(&content) {
|
|
Ok(val) => {
|
|
let val_str = format!("{}", val);
|
|
if val_str == expected {
|
|
results.push(TestResult { name, success: true, message: format!("OK: {}", val_str) });
|
|
} else {
|
|
results.push(TestResult { name, success: false, message: format!("Expected {}, got {}", expected, val_str) });
|
|
}
|
|
}
|
|
Err(e) => results.push(TestResult { name, success: false, message: format!("Error: {}", e) }),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
results
|
|
}
|
|
|
|
pub fn run_benchmarks(update: bool) -> Vec<BenchmarkResult> {
|
|
let mut results = Vec::new();
|
|
let entries = fs::read_dir("examples").unwrap();
|
|
let env = Environment::new();
|
|
let is_release = !cfg!(debug_assertions);
|
|
|
|
for entry in entries.filter_map(|e| e.ok()) {
|
|
let path = entry.path();
|
|
if path.extension().map_or(false, |ext| ext == "myc") {
|
|
let content = fs::read_to_string(&path).unwrap();
|
|
let name = path.file_name().unwrap().to_string_lossy().to_string();
|
|
|
|
let baseline_pattern = Regex::new(r";; Benchmark: ([\d\.]+\w+)").unwrap();
|
|
let baseline_match = baseline_pattern.captures(&content);
|
|
|
|
// Measure
|
|
let mut runs = Vec::new();
|
|
for _ in 0..100 {
|
|
let start = Instant::now();
|
|
let _ = env.run_script(&content);
|
|
runs.push(start.elapsed());
|
|
}
|
|
runs.sort();
|
|
let median = runs[runs.len() / 2];
|
|
|
|
if update {
|
|
let new_val = format_duration(median);
|
|
let updated_content = if let Some(m) = baseline_match {
|
|
content.replace(m.get(0).unwrap().as_str(), &format!(";; Benchmark: {}", new_val))
|
|
} else {
|
|
format!(";; Benchmark: {}
|
|
{}", new_val, content)
|
|
};
|
|
fs::write(&path, updated_content).unwrap();
|
|
results.push(BenchmarkResult { name, median, baseline: None, diff_pct: None, status: format!("UPDATED: {}", new_val) });
|
|
} else {
|
|
if let Some(m) = baseline_match {
|
|
let baseline_str = m.get(1).unwrap().as_str();
|
|
let baseline = parse_duration(baseline_str).unwrap();
|
|
let diff = (median.as_nanos() as f64 / baseline.as_nanos() as f64) - 1.0;
|
|
|
|
let threshold = if is_release { 0.10 } else { 0.25 };
|
|
let status = if median > baseline && diff > threshold { "FAILED" } else { "OK" };
|
|
|
|
results.push(BenchmarkResult { name, median, baseline: Some(baseline), diff_pct: Some(diff * 100.0), status: status.to_string() });
|
|
} else {
|
|
results.push(BenchmarkResult { name, median, baseline: None, diff_pct: None, status: "MISSING BASELINE".to_string() });
|
|
}
|
|
}
|
|
}
|
|
}
|
|
results
|
|
}
|
|
|
|
fn format_duration(d: Duration) -> String {
|
|
if d.as_nanos() < 1000 { format!("{}ns", d.as_nanos()) }
|
|
else if d.as_micros() < 1000 { format!("{:.1}us", d.as_nanos() as f64 / 1000.0) }
|
|
else if d.as_millis() < 1000 { format!("{:.1}ms", d.as_micros() as f64 / 1000.0) }
|
|
else { format!("{:.1}s", d.as_millis() as f64 / 1000.0) }
|
|
}
|
|
|
|
fn parse_duration(s: &str) -> Option<Duration> {
|
|
let re = Regex::new(r"([\d\.]+)(\w+)").unwrap();
|
|
let caps = re.captures(s)?;
|
|
let val: f64 = caps[1].parse().ok()?;
|
|
let unit = &caps[2];
|
|
match unit {
|
|
"ns" => Some(Duration::from_nanos(val as u64)),
|
|
"us" => Some(Duration::from_nanos((val * 1000.0) as u64)),
|
|
"ms" => Some(Duration::from_nanos((val * 1_000_000.0) as u64)),
|
|
"s" => Some(Duration::from_nanos((val * 1_000_000_000.0) as u64)),
|
|
_ => None,
|
|
}
|
|
}
|