Update benchmark timings and repeats

This commit updates the benchmark timings and repeat counts for various
example files. The `tester.rs` script has been modified to:

- Correctly parse and use the `Benchmark-Repeat` directive.
- Implement an adaptive sampling mechanism for more accurate median
  calculation, especially for long-running benchmarks.
- Improve the logic for updating and inserting benchmark and repeat
  lines in example files.
- Handle potential compilation and runtime errors during benchmark
  execution.
This commit is contained in:
Michael Schimmel
2026-02-20 17:11:13 +01:00
parent 87259584ee
commit 9072cfaa14
16 changed files with 195 additions and 103 deletions
+2 -1
View File
@@ -1,5 +1,6 @@
;; Closure Scope Test ;; Closure Scope Test
;; Benchmark: 800ns ;; Benchmark: 780ns
;; Benchmark-Repeat: 2602
;; Output: 15 ;; Output: 15
(do (do
(def make-adder (fn [x] (def make-adder (fn [x]
+2 -1
View File
@@ -1,5 +1,6 @@
;; Complex Data Structure Test ;; Complex Data Structure Test
;; Benchmark: 51.4us ;; Benchmark: 52.9us
;; Benchmark-Repeat: 39
;; Output: {:name "Fibonacci", :input 10, :output 55} ;; Output: {:name "Fibonacci", :input 10, :output 55}
(do (do
(def fib (fn [n] (def fib (fn [n]
+2
View File
@@ -1,3 +1,5 @@
;; Benchmark: 2.6us
;; Benchmark-Repeat: 788
;; Comprehensive Destructuring Test ;; Comprehensive Destructuring Test
;; Covers: Nested tuples, mixed params, dynamic passing ;; Covers: Nested tuples, mixed params, dynamic passing
+2 -1
View File
@@ -1,4 +1,5 @@
;; Benchmark: 1.6us ;; Benchmark: 1.8us
;; Benchmark-Repeat: 1103
;; Output: 36 ;; Output: 36
(do (do
;; Excessive capture test ;; Excessive capture test
+2 -1
View File
@@ -1,5 +1,6 @@
;; Fibonacci Recursive ;; Fibonacci Recursive
;; Benchmark: 49.7us ;; Benchmark: 52.7us
;; Benchmark-Repeat: 38
;; Output: 55 ;; Output: 55
(do (do
(def fib (fn [n] (def fib (fn [n]
+2 -1
View File
@@ -1,5 +1,6 @@
;; Higher-Order Function Example ;; Higher-Order Function Example
;; Benchmark: 600ns ;; Benchmark: 725ns
;; Benchmark-Repeat: 2768
;; Output: 25 ;; Output: 25
(do (do
(def apply (fn [f x] (f x))) (def apply (fn [f x] (f x)))
+2 -1
View File
@@ -1,4 +1,5 @@
;; Benchmark: 400ns ;; Benchmark: 484ns
;; Benchmark-Repeat: 4144
;; Financial DSL Macro example ;; Financial DSL Macro example
;; Demonstrates generating complex records from simple parameters. ;; Demonstrates generating complex records from simple parameters.
;; Output: {:price 100, :size 2, :total 200} ;; Output: {:price 100, :size 2, :total 200}
+2 -1
View File
@@ -1,4 +1,5 @@
;; Benchmark: 400ns ;; Benchmark: 448ns
;; Benchmark-Repeat: 4488
;; Output: 5 ;; Output: 5
(do (do
(def y 1) (def y 1)
+2 -1
View File
@@ -1,4 +1,5 @@
;; Benchmark: 300ns ;; Benchmark: 429ns
;; Benchmark-Repeat: 4712
;; Nested Macro and Arithmetic example ;; Nested Macro and Arithmetic example
;; Output: 81 ;; Output: 81
+2 -1
View File
@@ -1,4 +1,5 @@
;; Benchmark: 300ns ;; Benchmark: 320ns
;; Benchmark-Repeat: 6238
;; Macro Splicing example ;; Macro Splicing example
;; Demonstrates unrolling a list into another list. ;; Demonstrates unrolling a list into another list.
;; Output: [0 1 2 3 4] ;; Output: [0 1 2 3 4]
+2 -1
View File
@@ -1,4 +1,5 @@
;; Benchmark: 200ns ;; Benchmark: 209ns
;; Benchmark-Repeat: 9540
;; Classic "unless" macro example ;; Classic "unless" macro example
;; Demonstrates simple template substitution. ;; Demonstrates simple template substitution.
;; Output: 42 ;; Output: 42
+2 -1
View File
@@ -1,6 +1,7 @@
;; Record to Tuple Mapping Test ;; Record to Tuple Mapping Test
;; Output: 30 ;; Output: 30
;; Benchmark: 700ns ;; Benchmark: 755ns
;; Benchmark-Repeat: 2652
(do (do
(def rec {:x 10 :y 20}) (def rec {:x 10 :y 20})
+2 -1
View File
@@ -2,7 +2,8 @@
;; This test calls a function that destructures a record 100.000 times. ;; This test calls a function that destructures a record 100.000 times.
;; Current implementation is now zero-allocation for destructuring. ;; Current implementation is now zero-allocation for destructuring.
;; Output: {:a 1, :b 2} ;; Output: {:a 1, :b 2}
;; Benchmark: 5.3ms ;; Benchmark: 5.2ms
(do (do
(def process (fn [[x y]] (def process (fn [[x y]]
(+ x y))) (+ x y)))
+2 -1
View File
@@ -2,7 +2,8 @@
;; heavily recursive, benefits significantly from specialization of integer arithmetic ;; heavily recursive, benefits significantly from specialization of integer arithmetic
;; and direct function calls (skipping dynamic dispatch). ;; and direct function calls (skipping dynamic dispatch).
;; Output: 5 ;; Output: 5
;; Benchmark: 461.4us
;; Benchmark-Repeat: 6 ;; Benchmark-Repeat: 6
(do (do
+2 -1
View File
@@ -1,4 +1,5 @@
;; Benchmark: 1.9us ;; Benchmark: 1.7us
;; Benchmark-Repeat: 1196
;; Demonstration of N-dimensional Tuples, Vectors, and Matrices ;; Demonstration of N-dimensional Tuples, Vectors, and Matrices
;; Based on docs/Tupel 1.md ;; Based on docs/Tupel 1.md
;; ;;
+165 -89
View File
@@ -68,103 +68,179 @@ pub fn run_benchmarks(update: bool) -> Vec<BenchmarkResult> {
let entries = fs::read_dir("examples").unwrap(); let entries = fs::read_dir("examples").unwrap();
let is_release = !cfg!(debug_assertions); let is_release = !cfg!(debug_assertions);
let baseline_re = Regex::new(r";; Benchmark: ([\d\.]+\w+)").unwrap(); let baseline_re = Regex::new(r";; Benchmark: ([\d\.]+\w+)").unwrap();
let repeat_re = Regex::new(r";; Benchmark-Repeat: (\d+)").unwrap();
for entry in entries.filter_map(|e| e.ok()) { for entry in entries.filter_map(|e| e.ok()) {
let env = Environment::new(); // Fresh environment per benchmark for isolation
let path = entry.path(); let path = entry.path();
if path.extension().is_some_and(|ext| ext == "myc") { if !path.extension().is_some_and(|ext| ext == "myc") {
let content = fs::read_to_string(&path).unwrap(); continue;
let name = path.file_name().unwrap().to_string_lossy().to_string(); }
let baseline_match = baseline_re.captures(&content); let content = fs::read_to_string(&path).unwrap();
let name = path.file_name().unwrap().to_string_lossy().to_string();
// Prepare: Compile and Link once outside the measurement loop let baseline_match = baseline_re.captures(&content);
let linked_node = match env.compile(&content).map(|c| env.link(c)) { let repeat_match = repeat_re.captures(&content);
Ok(node) => node,
Err(e) => { let mut repeats = repeat_match
results.push(BenchmarkResult { .and_then(|m| m.get(1))
name, .and_then(|m| m.as_str().parse::<u32>().ok())
median: Duration::ZERO, .unwrap_or(1);
baseline: None,
diff_pct: None, // Compile once for this file (symbols/types are compatible with all fresh environments)
status: format!("COMPILE ERROR: {}", e), let initial_env = Environment::new();
}); let compiled_once = match initial_env.compile(&content) {
continue; Ok(c) => c,
Err(e) => {
results.push(BenchmarkResult {
name,
median: Duration::ZERO,
baseline: None,
diff_pct: None,
status: format!("COMPILE ERROR: {}", e),
});
continue;
}
};
// Helper to measure sum of VM execution times over N fresh environments
let measure_sum = |n: u32, node: &crate::ast::compiler::TypedNode| -> Result<Duration, String> {
let mut total = Duration::ZERO;
for _ in 0..n {
let env = Environment::new();
// Link is still required per environment as it populates registries
let linked = env.link(node.clone());
let start = Instant::now();
let _ = env.run(&linked)?;
total += start.elapsed();
}
Ok(total)
};
if update {
repeats = 1;
loop {
match measure_sum(repeats, &compiled_once) {
Ok(total) => {
if total >= Duration::from_millis(2) || repeats >= 100_000 {
break;
}
let nanos = total.as_nanos().max(1) as f64;
let factor = 2_000_000.0 / nanos;
repeats = (repeats as f64 * factor).ceil() as u32;
repeats = repeats.max(repeats + 1);
}
Err(e) => {
results.push(BenchmarkResult {
name: name.clone(),
median: Duration::ZERO,
baseline: None,
diff_pct: None,
status: format!("ERROR: {}", e),
});
break;
}
} }
}
if results.last().map_or(false, |r| r.name == name) { continue; }
}
let mut runs = Vec::new();
let mut error = None;
// Adaptive samples: High repeats need fewer samples for stable median
let num_samples = if repeats > 1000 { 10 } else if repeats > 100 { 30 } else { 100 };
for _ in 0..num_samples {
match measure_sum(repeats, &compiled_once) {
Ok(d) => runs.push(d),
Err(e) => {
error = Some(e);
break;
}
}
}
if let Some(e) = error {
results.push(BenchmarkResult {
name,
median: Duration::ZERO,
baseline: None,
diff_pct: None,
status: format!("ERROR: {}", e),
});
continue;
}
runs.sort();
let median_total = runs[runs.len() / 2];
let median_single = median_total / repeats;
if update {
let new_val = format_duration(median_single);
let mut updated_content = content.clone();
// 1. Update/Insert Benchmark
let bench_line = format!(";; Benchmark: {}", new_val);
if let Some(m) = baseline_match {
updated_content = updated_content.replace(m.get(0).unwrap().as_str(), &bench_line);
} else {
updated_content = format!("{}\n{}", bench_line, updated_content);
}
// 2. Update/Insert/Remove Benchmark-Repeat
let repeat_line = if repeats > 1 { Some(format!(";; Benchmark-Repeat: {}", repeats)) } else { None };
let current_repeat_str = repeat_re.captures(&updated_content).map(|m| m.get(0).unwrap().as_str().to_string());
if let Some(line) = repeat_line {
if let Some(old_line) = current_repeat_str {
updated_content = updated_content.replace(&old_line, &line);
} else if let Some(m) = baseline_re.captures(&updated_content) {
let b_str = m.get(0).unwrap().as_str().to_string();
if let Some(pos) = updated_content.find(&b_str) {
updated_content.insert_str(pos + b_str.len(), &format!("\n{}", line));
}
}
} else if let Some(old_line) = current_repeat_str {
updated_content = updated_content.replace(&format!("{}\n", old_line), "");
updated_content = updated_content.replace(&old_line, "");
}
fs::write(&path, updated_content).unwrap();
results.push(BenchmarkResult {
name,
median: median_single,
baseline: None,
diff_pct: None,
status: format!("UPDATED: {}", new_val),
});
} else if let Some(m) = baseline_match {
let baseline_str = m.get(1).unwrap().as_str();
let baseline = parse_duration(baseline_str).unwrap();
let diff = (median_single.as_nanos() as f64 / baseline.as_nanos() as f64) - 1.0;
let threshold = if is_release { 0.15 } else { 0.5 };
let status = if median_single > baseline && diff > threshold {
"FAILED"
} else {
"OK"
}; };
// Measure: Only the VM execution results.push(BenchmarkResult {
let mut runs = Vec::new(); name,
for _ in 0..100 { median: median_single,
let start = Instant::now(); baseline: Some(baseline),
let _ = env.run(&linked_node); diff_pct: Some(diff * 100.0),
runs.push(start.elapsed()); status: status.to_string(),
} });
runs.sort(); } else {
let median = runs[runs.len() / 2]; results.push(BenchmarkResult {
name,
if update { median: median_single,
let new_val = format_duration(median); baseline: None,
let updated_content = if let Some(m) = baseline_match { diff_pct: None,
content.replace( status: "MISSING BASELINE".to_string(),
m.get(0).unwrap().as_str(), });
&format!(";; Benchmark: {}", new_val),
)
} else {
format!(
";; Benchmark: {}
{}",
new_val, content
)
};
fs::write(&path, updated_content).unwrap();
results.push(BenchmarkResult {
name,
median,
baseline: None,
diff_pct: None,
status: format!("UPDATED: {}", new_val),
});
} else if let Some(m) = baseline_match {
let baseline_str = m.get(1).unwrap().as_str();
let baseline = parse_duration(baseline_str).unwrap();
if baseline < Duration::from_micros(1) {
results.push(BenchmarkResult {
name,
median,
baseline: Some(baseline),
diff_pct: None,
status: "SKIPPED".to_string(),
});
continue;
}
let diff = (median.as_nanos() as f64 / baseline.as_nanos() as f64) - 1.0;
let threshold = if is_release { 0.15 } else { 0.5 };
let status = if median > baseline && diff > threshold {
"FAILED"
} else {
"OK"
};
results.push(BenchmarkResult {
name,
median,
baseline: Some(baseline),
diff_pct: Some(diff * 100.0),
status: status.to_string(),
});
} else {
results.push(BenchmarkResult {
name,
median,
baseline: None,
diff_pct: None,
status: "MISSING BASELINE".to_string(),
});
}
} }
} }
results results