Update benchmark timings and repeats
This commit updates the benchmark timings and repeat counts for various example files. The `tester.rs` script has been modified to: - Correctly parse and use the `Benchmark-Repeat` directive. - Implement an adaptive sampling mechanism for more accurate median calculation, especially for long-running benchmarks. - Improve the logic for updating and inserting benchmark and repeat lines in example files. - Handle potential compilation and runtime errors during benchmark execution.
This commit is contained in:
Michael Schimmel
+165
-89
@@ -68,103 +68,179 @@ pub fn run_benchmarks(update: bool) -> Vec<BenchmarkResult> {
|
||||
let entries = fs::read_dir("examples").unwrap();
|
||||
let is_release = !cfg!(debug_assertions);
|
||||
let baseline_re = Regex::new(r";; Benchmark: ([\d\.]+\w+)").unwrap();
|
||||
let repeat_re = Regex::new(r";; Benchmark-Repeat: (\d+)").unwrap();
|
||||
|
||||
for entry in entries.filter_map(|e| e.ok()) {
|
||||
let env = Environment::new(); // Fresh environment per benchmark for isolation
|
||||
let path = entry.path();
|
||||
if path.extension().is_some_and(|ext| ext == "myc") {
|
||||
let content = fs::read_to_string(&path).unwrap();
|
||||
let name = path.file_name().unwrap().to_string_lossy().to_string();
|
||||
if !path.extension().is_some_and(|ext| ext == "myc") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let baseline_match = baseline_re.captures(&content);
|
||||
let content = fs::read_to_string(&path).unwrap();
|
||||
let name = path.file_name().unwrap().to_string_lossy().to_string();
|
||||
|
||||
// Prepare: Compile and Link once outside the measurement loop
|
||||
let linked_node = match env.compile(&content).map(|c| env.link(c)) {
|
||||
Ok(node) => node,
|
||||
Err(e) => {
|
||||
results.push(BenchmarkResult {
|
||||
name,
|
||||
median: Duration::ZERO,
|
||||
baseline: None,
|
||||
diff_pct: None,
|
||||
status: format!("COMPILE ERROR: {}", e),
|
||||
});
|
||||
continue;
|
||||
let baseline_match = baseline_re.captures(&content);
|
||||
let repeat_match = repeat_re.captures(&content);
|
||||
|
||||
let mut repeats = repeat_match
|
||||
.and_then(|m| m.get(1))
|
||||
.and_then(|m| m.as_str().parse::<u32>().ok())
|
||||
.unwrap_or(1);
|
||||
|
||||
// Compile once for this file (symbols/types are compatible with all fresh environments)
|
||||
let initial_env = Environment::new();
|
||||
let compiled_once = match initial_env.compile(&content) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
results.push(BenchmarkResult {
|
||||
name,
|
||||
median: Duration::ZERO,
|
||||
baseline: None,
|
||||
diff_pct: None,
|
||||
status: format!("COMPILE ERROR: {}", e),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
// Helper to measure sum of VM execution times over N fresh environments
|
||||
let measure_sum = |n: u32, node: &crate::ast::compiler::TypedNode| -> Result<Duration, String> {
|
||||
let mut total = Duration::ZERO;
|
||||
for _ in 0..n {
|
||||
let env = Environment::new();
|
||||
// Link is still required per environment as it populates registries
|
||||
let linked = env.link(node.clone());
|
||||
let start = Instant::now();
|
||||
let _ = env.run(&linked)?;
|
||||
total += start.elapsed();
|
||||
}
|
||||
Ok(total)
|
||||
};
|
||||
|
||||
if update {
|
||||
repeats = 1;
|
||||
loop {
|
||||
match measure_sum(repeats, &compiled_once) {
|
||||
Ok(total) => {
|
||||
if total >= Duration::from_millis(2) || repeats >= 100_000 {
|
||||
break;
|
||||
}
|
||||
let nanos = total.as_nanos().max(1) as f64;
|
||||
let factor = 2_000_000.0 / nanos;
|
||||
repeats = (repeats as f64 * factor).ceil() as u32;
|
||||
repeats = repeats.max(repeats + 1);
|
||||
}
|
||||
Err(e) => {
|
||||
results.push(BenchmarkResult {
|
||||
name: name.clone(),
|
||||
median: Duration::ZERO,
|
||||
baseline: None,
|
||||
diff_pct: None,
|
||||
status: format!("ERROR: {}", e),
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if results.last().map_or(false, |r| r.name == name) { continue; }
|
||||
}
|
||||
|
||||
let mut runs = Vec::new();
|
||||
let mut error = None;
|
||||
// Adaptive samples: High repeats need fewer samples for stable median
|
||||
let num_samples = if repeats > 1000 { 10 } else if repeats > 100 { 30 } else { 100 };
|
||||
|
||||
for _ in 0..num_samples {
|
||||
match measure_sum(repeats, &compiled_once) {
|
||||
Ok(d) => runs.push(d),
|
||||
Err(e) => {
|
||||
error = Some(e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(e) = error {
|
||||
results.push(BenchmarkResult {
|
||||
name,
|
||||
median: Duration::ZERO,
|
||||
baseline: None,
|
||||
diff_pct: None,
|
||||
status: format!("ERROR: {}", e),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
runs.sort();
|
||||
let median_total = runs[runs.len() / 2];
|
||||
let median_single = median_total / repeats;
|
||||
|
||||
if update {
|
||||
let new_val = format_duration(median_single);
|
||||
let mut updated_content = content.clone();
|
||||
|
||||
// 1. Update/Insert Benchmark
|
||||
let bench_line = format!(";; Benchmark: {}", new_val);
|
||||
if let Some(m) = baseline_match {
|
||||
updated_content = updated_content.replace(m.get(0).unwrap().as_str(), &bench_line);
|
||||
} else {
|
||||
updated_content = format!("{}\n{}", bench_line, updated_content);
|
||||
}
|
||||
|
||||
// 2. Update/Insert/Remove Benchmark-Repeat
|
||||
let repeat_line = if repeats > 1 { Some(format!(";; Benchmark-Repeat: {}", repeats)) } else { None };
|
||||
let current_repeat_str = repeat_re.captures(&updated_content).map(|m| m.get(0).unwrap().as_str().to_string());
|
||||
|
||||
if let Some(line) = repeat_line {
|
||||
if let Some(old_line) = current_repeat_str {
|
||||
updated_content = updated_content.replace(&old_line, &line);
|
||||
} else if let Some(m) = baseline_re.captures(&updated_content) {
|
||||
let b_str = m.get(0).unwrap().as_str().to_string();
|
||||
if let Some(pos) = updated_content.find(&b_str) {
|
||||
updated_content.insert_str(pos + b_str.len(), &format!("\n{}", line));
|
||||
}
|
||||
}
|
||||
} else if let Some(old_line) = current_repeat_str {
|
||||
updated_content = updated_content.replace(&format!("{}\n", old_line), "");
|
||||
updated_content = updated_content.replace(&old_line, "");
|
||||
}
|
||||
|
||||
fs::write(&path, updated_content).unwrap();
|
||||
results.push(BenchmarkResult {
|
||||
name,
|
||||
median: median_single,
|
||||
baseline: None,
|
||||
diff_pct: None,
|
||||
status: format!("UPDATED: {}", new_val),
|
||||
});
|
||||
} else if let Some(m) = baseline_match {
|
||||
let baseline_str = m.get(1).unwrap().as_str();
|
||||
let baseline = parse_duration(baseline_str).unwrap();
|
||||
|
||||
let diff = (median_single.as_nanos() as f64 / baseline.as_nanos() as f64) - 1.0;
|
||||
|
||||
let threshold = if is_release { 0.15 } else { 0.5 };
|
||||
let status = if median_single > baseline && diff > threshold {
|
||||
"FAILED"
|
||||
} else {
|
||||
"OK"
|
||||
};
|
||||
|
||||
// Measure: Only the VM execution
|
||||
let mut runs = Vec::new();
|
||||
for _ in 0..100 {
|
||||
let start = Instant::now();
|
||||
let _ = env.run(&linked_node);
|
||||
runs.push(start.elapsed());
|
||||
}
|
||||
runs.sort();
|
||||
let median = runs[runs.len() / 2];
|
||||
|
||||
if update {
|
||||
let new_val = format_duration(median);
|
||||
let updated_content = if let Some(m) = baseline_match {
|
||||
content.replace(
|
||||
m.get(0).unwrap().as_str(),
|
||||
&format!(";; Benchmark: {}", new_val),
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
";; Benchmark: {}
|
||||
{}",
|
||||
new_val, content
|
||||
)
|
||||
};
|
||||
fs::write(&path, updated_content).unwrap();
|
||||
results.push(BenchmarkResult {
|
||||
name,
|
||||
median,
|
||||
baseline: None,
|
||||
diff_pct: None,
|
||||
status: format!("UPDATED: {}", new_val),
|
||||
});
|
||||
} else if let Some(m) = baseline_match {
|
||||
let baseline_str = m.get(1).unwrap().as_str();
|
||||
let baseline = parse_duration(baseline_str).unwrap();
|
||||
|
||||
if baseline < Duration::from_micros(1) {
|
||||
results.push(BenchmarkResult {
|
||||
name,
|
||||
median,
|
||||
baseline: Some(baseline),
|
||||
diff_pct: None,
|
||||
status: "SKIPPED".to_string(),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
let diff = (median.as_nanos() as f64 / baseline.as_nanos() as f64) - 1.0;
|
||||
|
||||
let threshold = if is_release { 0.15 } else { 0.5 };
|
||||
let status = if median > baseline && diff > threshold {
|
||||
"FAILED"
|
||||
} else {
|
||||
"OK"
|
||||
};
|
||||
|
||||
results.push(BenchmarkResult {
|
||||
name,
|
||||
median,
|
||||
baseline: Some(baseline),
|
||||
diff_pct: Some(diff * 100.0),
|
||||
status: status.to_string(),
|
||||
});
|
||||
} else {
|
||||
results.push(BenchmarkResult {
|
||||
name,
|
||||
median,
|
||||
baseline: None,
|
||||
diff_pct: None,
|
||||
status: "MISSING BASELINE".to_string(),
|
||||
});
|
||||
}
|
||||
results.push(BenchmarkResult {
|
||||
name,
|
||||
median: median_single,
|
||||
baseline: Some(baseline),
|
||||
diff_pct: Some(diff * 100.0),
|
||||
status: status.to_string(),
|
||||
});
|
||||
} else {
|
||||
results.push(BenchmarkResult {
|
||||
name,
|
||||
median: median_single,
|
||||
baseline: None,
|
||||
diff_pct: None,
|
||||
status: "MISSING BASELINE".to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
results
|
||||
|
||||
Reference in New Issue
Block a user