From 9072cfaa14d2196ee83894a1bebb036c2f9bebf5 Mon Sep 17 00:00:00 2001 From: Michael Schimmel Date: Fri, 20 Feb 2026 17:11:13 +0100 Subject: [PATCH] Update benchmark timings and repeats This commit updates the benchmark timings and repeat counts for various example files. The `tester.rs` script has been modified to: - Correctly parse and use the `Benchmark-Repeat` directive. - Implement an adaptive sampling mechanism for more accurate median calculation, especially for long-running benchmarks. - Improve the logic for updating and inserting benchmark and repeat lines in example files. - Handle potential compilation and runtime errors during benchmark execution. --- examples/closure.myc | 3 +- examples/data.myc | 3 +- examples/destructuring.myc | 2 + examples/extreme_capture.myc | 3 +- examples/fib.myc | 3 +- examples/hof.myc | 3 +- examples/macro_financial.myc | 3 +- examples/macro_hygiene.myc | 3 +- examples/macro_power.myc | 3 +- examples/macro_splice.myc | 3 +- examples/macro_unless.myc | 3 +- examples/record_to_tuple.myc | 3 +- examples/record_unpack.myc | 3 +- examples/tak.myc | 3 +- examples/tuples.myc | 3 +- src/utils/tester.rs | 254 +++++++++++++++++++++++------------ 16 files changed, 195 insertions(+), 103 deletions(-) diff --git a/examples/closure.myc b/examples/closure.myc index 5beb5a4..ee402b2 100644 --- a/examples/closure.myc +++ b/examples/closure.myc @@ -1,5 +1,6 @@ ;; Closure Scope Test -;; Benchmark: 800ns +;; Benchmark: 780ns +;; Benchmark-Repeat: 2602 ;; Output: 15 (do (def make-adder (fn [x] diff --git a/examples/data.myc b/examples/data.myc index 82903e5..24d0cb2 100644 --- a/examples/data.myc +++ b/examples/data.myc @@ -1,5 +1,6 @@ ;; Complex Data Structure Test -;; Benchmark: 51.4us +;; Benchmark: 52.9us +;; Benchmark-Repeat: 39 ;; Output: {:name "Fibonacci", :input 10, :output 55} (do (def fib (fn [n] diff --git a/examples/destructuring.myc b/examples/destructuring.myc index b4e9043..25e3177 100644 --- a/examples/destructuring.myc +++ b/examples/destructuring.myc @@ -1,3 +1,5 @@ +;; Benchmark: 2.6us +;; Benchmark-Repeat: 788 ;; Comprehensive Destructuring Test ;; Covers: Nested tuples, mixed params, dynamic passing diff --git a/examples/extreme_capture.myc b/examples/extreme_capture.myc index e86b19e..d600a76 100644 --- a/examples/extreme_capture.myc +++ b/examples/extreme_capture.myc @@ -1,4 +1,5 @@ -;; Benchmark: 1.6us +;; Benchmark: 1.8us +;; Benchmark-Repeat: 1103 ;; Output: 36 (do ;; Excessive capture test diff --git a/examples/fib.myc b/examples/fib.myc index 0e07f46..1a6bf14 100644 --- a/examples/fib.myc +++ b/examples/fib.myc @@ -1,5 +1,6 @@ ;; Fibonacci Recursive -;; Benchmark: 49.7us +;; Benchmark: 52.7us +;; Benchmark-Repeat: 38 ;; Output: 55 (do (def fib (fn [n] diff --git a/examples/hof.myc b/examples/hof.myc index 35d16de..0c81837 100644 --- a/examples/hof.myc +++ b/examples/hof.myc @@ -1,5 +1,6 @@ ;; Higher-Order Function Example -;; Benchmark: 600ns +;; Benchmark: 725ns +;; Benchmark-Repeat: 2768 ;; Output: 25 (do (def apply (fn [f x] (f x))) diff --git a/examples/macro_financial.myc b/examples/macro_financial.myc index f6472d7..e8bed14 100644 --- a/examples/macro_financial.myc +++ b/examples/macro_financial.myc @@ -1,4 +1,5 @@ -;; Benchmark: 400ns +;; Benchmark: 484ns +;; Benchmark-Repeat: 4144 ;; Financial DSL Macro example ;; Demonstrates generating complex records from simple parameters. ;; Output: {:price 100, :size 2, :total 200} diff --git a/examples/macro_hygiene.myc b/examples/macro_hygiene.myc index a32336f..4db70fb 100644 --- a/examples/macro_hygiene.myc +++ b/examples/macro_hygiene.myc @@ -1,4 +1,5 @@ -;; Benchmark: 400ns +;; Benchmark: 448ns +;; Benchmark-Repeat: 4488 ;; Output: 5 (do (def y 1) diff --git a/examples/macro_power.myc b/examples/macro_power.myc index c1c85b3..e0ac61a 100644 --- a/examples/macro_power.myc +++ b/examples/macro_power.myc @@ -1,4 +1,5 @@ -;; Benchmark: 300ns +;; Benchmark: 429ns +;; Benchmark-Repeat: 4712 ;; Nested Macro and Arithmetic example ;; Output: 81 diff --git a/examples/macro_splice.myc b/examples/macro_splice.myc index 9a2a931..893aeff 100644 --- a/examples/macro_splice.myc +++ b/examples/macro_splice.myc @@ -1,4 +1,5 @@ -;; Benchmark: 300ns +;; Benchmark: 320ns +;; Benchmark-Repeat: 6238 ;; Macro Splicing example ;; Demonstrates unrolling a list into another list. ;; Output: [0 1 2 3 4] diff --git a/examples/macro_unless.myc b/examples/macro_unless.myc index 63d8e0e..6584678 100644 --- a/examples/macro_unless.myc +++ b/examples/macro_unless.myc @@ -1,4 +1,5 @@ -;; Benchmark: 200ns +;; Benchmark: 209ns +;; Benchmark-Repeat: 9540 ;; Classic "unless" macro example ;; Demonstrates simple template substitution. ;; Output: 42 diff --git a/examples/record_to_tuple.myc b/examples/record_to_tuple.myc index b2028c0..bffc631 100644 --- a/examples/record_to_tuple.myc +++ b/examples/record_to_tuple.myc @@ -1,6 +1,7 @@ ;; Record to Tuple Mapping Test ;; Output: 30 -;; Benchmark: 700ns +;; Benchmark: 755ns +;; Benchmark-Repeat: 2652 (do (def rec {:x 10 :y 20}) diff --git a/examples/record_unpack.myc b/examples/record_unpack.myc index 8bbaa1c..e02952a 100644 --- a/examples/record_unpack.myc +++ b/examples/record_unpack.myc @@ -2,7 +2,8 @@ ;; This test calls a function that destructures a record 100.000 times. ;; Current implementation is now zero-allocation for destructuring. ;; Output: {:a 1, :b 2} -;; Benchmark: 5.3ms +;; Benchmark: 5.2ms + (do (def process (fn [[x y]] (+ x y))) diff --git a/examples/tak.myc b/examples/tak.myc index 3619faa..b358785 100644 --- a/examples/tak.myc +++ b/examples/tak.myc @@ -2,7 +2,8 @@ ;; heavily recursive, benefits significantly from specialization of integer arithmetic ;; and direct function calls (skipping dynamic dispatch). ;; Output: 5 - ;; Benchmark: 467.7us + ;; Benchmark: 461.4us +;; Benchmark-Repeat: 6 (do (def tak (fn [x y z] diff --git a/examples/tuples.myc b/examples/tuples.myc index eb2dc15..70897dd 100644 --- a/examples/tuples.myc +++ b/examples/tuples.myc @@ -1,4 +1,5 @@ -;; Benchmark: 1.9us +;; Benchmark: 1.7us +;; Benchmark-Repeat: 1196 ;; Demonstration of N-dimensional Tuples, Vectors, and Matrices ;; Based on docs/Tupel 1.md ;; diff --git a/src/utils/tester.rs b/src/utils/tester.rs index c61849b..a3d045c 100644 --- a/src/utils/tester.rs +++ b/src/utils/tester.rs @@ -68,103 +68,179 @@ pub fn run_benchmarks(update: bool) -> Vec { let entries = fs::read_dir("examples").unwrap(); let is_release = !cfg!(debug_assertions); let baseline_re = Regex::new(r";; Benchmark: ([\d\.]+\w+)").unwrap(); + let repeat_re = Regex::new(r";; Benchmark-Repeat: (\d+)").unwrap(); for entry in entries.filter_map(|e| e.ok()) { - let env = Environment::new(); // Fresh environment per benchmark for isolation let path = entry.path(); - if path.extension().is_some_and(|ext| ext == "myc") { - let content = fs::read_to_string(&path).unwrap(); - let name = path.file_name().unwrap().to_string_lossy().to_string(); + if !path.extension().is_some_and(|ext| ext == "myc") { + continue; + } - let baseline_match = baseline_re.captures(&content); + let content = fs::read_to_string(&path).unwrap(); + let name = path.file_name().unwrap().to_string_lossy().to_string(); - // Prepare: Compile and Link once outside the measurement loop - let linked_node = match env.compile(&content).map(|c| env.link(c)) { - Ok(node) => node, - Err(e) => { - results.push(BenchmarkResult { - name, - median: Duration::ZERO, - baseline: None, - diff_pct: None, - status: format!("COMPILE ERROR: {}", e), - }); - continue; + let baseline_match = baseline_re.captures(&content); + let repeat_match = repeat_re.captures(&content); + + let mut repeats = repeat_match + .and_then(|m| m.get(1)) + .and_then(|m| m.as_str().parse::().ok()) + .unwrap_or(1); + + // Compile once for this file (symbols/types are compatible with all fresh environments) + let initial_env = Environment::new(); + let compiled_once = match initial_env.compile(&content) { + Ok(c) => c, + Err(e) => { + results.push(BenchmarkResult { + name, + median: Duration::ZERO, + baseline: None, + diff_pct: None, + status: format!("COMPILE ERROR: {}", e), + }); + continue; + } + }; + + // Helper to measure sum of VM execution times over N fresh environments + let measure_sum = |n: u32, node: &crate::ast::compiler::TypedNode| -> Result { + let mut total = Duration::ZERO; + for _ in 0..n { + let env = Environment::new(); + // Link is still required per environment as it populates registries + let linked = env.link(node.clone()); + let start = Instant::now(); + let _ = env.run(&linked)?; + total += start.elapsed(); + } + Ok(total) + }; + + if update { + repeats = 1; + loop { + match measure_sum(repeats, &compiled_once) { + Ok(total) => { + if total >= Duration::from_millis(2) || repeats >= 100_000 { + break; + } + let nanos = total.as_nanos().max(1) as f64; + let factor = 2_000_000.0 / nanos; + repeats = (repeats as f64 * factor).ceil() as u32; + repeats = repeats.max(repeats + 1); + } + Err(e) => { + results.push(BenchmarkResult { + name: name.clone(), + median: Duration::ZERO, + baseline: None, + diff_pct: None, + status: format!("ERROR: {}", e), + }); + break; + } } + } + if results.last().map_or(false, |r| r.name == name) { continue; } + } + + let mut runs = Vec::new(); + let mut error = None; + // Adaptive samples: High repeats need fewer samples for stable median + let num_samples = if repeats > 1000 { 10 } else if repeats > 100 { 30 } else { 100 }; + + for _ in 0..num_samples { + match measure_sum(repeats, &compiled_once) { + Ok(d) => runs.push(d), + Err(e) => { + error = Some(e); + break; + } + } + } + + if let Some(e) = error { + results.push(BenchmarkResult { + name, + median: Duration::ZERO, + baseline: None, + diff_pct: None, + status: format!("ERROR: {}", e), + }); + continue; + } + + runs.sort(); + let median_total = runs[runs.len() / 2]; + let median_single = median_total / repeats; + + if update { + let new_val = format_duration(median_single); + let mut updated_content = content.clone(); + + // 1. Update/Insert Benchmark + let bench_line = format!(";; Benchmark: {}", new_val); + if let Some(m) = baseline_match { + updated_content = updated_content.replace(m.get(0).unwrap().as_str(), &bench_line); + } else { + updated_content = format!("{}\n{}", bench_line, updated_content); + } + + // 2. Update/Insert/Remove Benchmark-Repeat + let repeat_line = if repeats > 1 { Some(format!(";; Benchmark-Repeat: {}", repeats)) } else { None }; + let current_repeat_str = repeat_re.captures(&updated_content).map(|m| m.get(0).unwrap().as_str().to_string()); + + if let Some(line) = repeat_line { + if let Some(old_line) = current_repeat_str { + updated_content = updated_content.replace(&old_line, &line); + } else if let Some(m) = baseline_re.captures(&updated_content) { + let b_str = m.get(0).unwrap().as_str().to_string(); + if let Some(pos) = updated_content.find(&b_str) { + updated_content.insert_str(pos + b_str.len(), &format!("\n{}", line)); + } + } + } else if let Some(old_line) = current_repeat_str { + updated_content = updated_content.replace(&format!("{}\n", old_line), ""); + updated_content = updated_content.replace(&old_line, ""); + } + + fs::write(&path, updated_content).unwrap(); + results.push(BenchmarkResult { + name, + median: median_single, + baseline: None, + diff_pct: None, + status: format!("UPDATED: {}", new_val), + }); + } else if let Some(m) = baseline_match { + let baseline_str = m.get(1).unwrap().as_str(); + let baseline = parse_duration(baseline_str).unwrap(); + + let diff = (median_single.as_nanos() as f64 / baseline.as_nanos() as f64) - 1.0; + + let threshold = if is_release { 0.15 } else { 0.5 }; + let status = if median_single > baseline && diff > threshold { + "FAILED" + } else { + "OK" }; - // Measure: Only the VM execution - let mut runs = Vec::new(); - for _ in 0..100 { - let start = Instant::now(); - let _ = env.run(&linked_node); - runs.push(start.elapsed()); - } - runs.sort(); - let median = runs[runs.len() / 2]; - - if update { - let new_val = format_duration(median); - let updated_content = if let Some(m) = baseline_match { - content.replace( - m.get(0).unwrap().as_str(), - &format!(";; Benchmark: {}", new_val), - ) - } else { - format!( - ";; Benchmark: {} -{}", - new_val, content - ) - }; - fs::write(&path, updated_content).unwrap(); - results.push(BenchmarkResult { - name, - median, - baseline: None, - diff_pct: None, - status: format!("UPDATED: {}", new_val), - }); - } else if let Some(m) = baseline_match { - let baseline_str = m.get(1).unwrap().as_str(); - let baseline = parse_duration(baseline_str).unwrap(); - - if baseline < Duration::from_micros(1) { - results.push(BenchmarkResult { - name, - median, - baseline: Some(baseline), - diff_pct: None, - status: "SKIPPED".to_string(), - }); - continue; - } - - let diff = (median.as_nanos() as f64 / baseline.as_nanos() as f64) - 1.0; - - let threshold = if is_release { 0.15 } else { 0.5 }; - let status = if median > baseline && diff > threshold { - "FAILED" - } else { - "OK" - }; - - results.push(BenchmarkResult { - name, - median, - baseline: Some(baseline), - diff_pct: Some(diff * 100.0), - status: status.to_string(), - }); - } else { - results.push(BenchmarkResult { - name, - median, - baseline: None, - diff_pct: None, - status: "MISSING BASELINE".to_string(), - }); - } + results.push(BenchmarkResult { + name, + median: median_single, + baseline: Some(baseline), + diff_pct: Some(diff * 100.0), + status: status.to_string(), + }); + } else { + results.push(BenchmarkResult { + name, + median: median_single, + baseline: None, + diff_pct: None, + status: "MISSING BASELINE".to_string(), + }); } } results