Brummel/RustAst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update benchmark timings and repeats

Browse Source 
This commit updates the benchmark timings and repeat counts for various
example files. The `tester.rs` script has been modified to:

- Correctly parse and use the `Benchmark-Repeat` directive.
- Implement an adaptive sampling mechanism for more accurate median
  calculation, especially for long-running benchmarks.
- Improve the logic for updating and inserting benchmark and repeat
  lines in example files.
- Handle potential compilation and runtime errors during benchmark
  execution.
This commit is contained in:
Michael Schimmel
2026-02-20 17:11:13 +01:00
parent 87259584ee
commit 9072cfaa14
16 changed files with 195 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/closure.myc
+2 -1
View File
@@ -1,5 +1,6 @@
;; Closure Scope Test
;; Benchmark: 800ns
;; Benchmark: 780ns
;; Benchmark-Repeat: 2602
;; Output: 15
(do
(def make-adder (fn [x]
examples/data.myc
+2 -1
View File
@@ -1,5 +1,6 @@
;; Complex Data Structure Test
;; Benchmark: 51.4us
;; Benchmark: 52.9us
;; Benchmark-Repeat: 39
;; Output: {:name "Fibonacci", :input 10, :output 55}
(do
(def fib (fn [n]
examples/destructuring.myc
+2
View File
@@ -1,3 +1,5 @@
;; Benchmark: 2.6us
;; Benchmark-Repeat: 788
;; Comprehensive Destructuring Test
;; Covers: Nested tuples, mixed params, dynamic passing
examples/extreme_capture.myc
+2 -1
View File
@@ -1,4 +1,5 @@
;; Benchmark: 1.6us
;; Benchmark: 1.8us
;; Benchmark-Repeat: 1103
;; Output: 36
(do
;; Excessive capture test
examples/fib.myc
+2 -1
View File
@@ -1,5 +1,6 @@
;; Fibonacci Recursive
;; Benchmark: 49.7us
;; Benchmark: 52.7us
;; Benchmark-Repeat: 38
;; Output: 55
(do
(def fib (fn [n]
examples/hof.myc
+2 -1
View File
@@ -1,5 +1,6 @@
;; Higher-Order Function Example
;; Benchmark: 600ns
;; Benchmark: 725ns
;; Benchmark-Repeat: 2768
;; Output: 25
(do
(def apply (fn [f x] (f x)))
examples/macro_financial.myc
+2 -1
View File
@@ -1,4 +1,5 @@
;; Benchmark: 400ns
;; Benchmark: 484ns
;; Benchmark-Repeat: 4144
;; Financial DSL Macro example
;; Demonstrates generating complex records from simple parameters.
;; Output: {:price 100, :size 2, :total 200}
examples/macro_hygiene.myc
+2 -1
View File
@@ -1,4 +1,5 @@
;; Benchmark: 400ns
;; Benchmark: 448ns
;; Benchmark-Repeat: 4488
;; Output: 5
(do
(def y 1)
examples/macro_power.myc
+2 -1
View File
@@ -1,4 +1,5 @@
;; Benchmark: 300ns
;; Benchmark: 429ns
;; Benchmark-Repeat: 4712
;; Nested Macro and Arithmetic example
;; Output: 81
examples/macro_splice.myc
+2 -1
View File
@@ -1,4 +1,5 @@
;; Benchmark: 300ns
;; Benchmark: 320ns
;; Benchmark-Repeat: 6238
;; Macro Splicing example
;; Demonstrates unrolling a list into another list.
;; Output: [0 1 2 3 4]
examples/macro_unless.myc
+2 -1
View File
@@ -1,4 +1,5 @@
;; Benchmark: 200ns
;; Benchmark: 209ns
;; Benchmark-Repeat: 9540
;; Classic "unless" macro example
;; Demonstrates simple template substitution.
;; Output: 42
examples/record_to_tuple.myc
+2 -1
View File
@@ -1,6 +1,7 @@
;; Record to Tuple Mapping Test
;; Output: 30
;; Benchmark: 700ns
;; Benchmark: 755ns
;; Benchmark-Repeat: 2652
(do
(def rec {:x 10 :y 20})
examples/record_unpack.myc
+2 -1
View File
@@ -2,7 +2,8 @@
;; This test calls a function that destructures a record 100.000 times.
;; Current implementation is now zero-allocation for destructuring.
;; Output: {:a 1, :b 2}
;; Benchmark: 5.3ms
;; Benchmark: 5.2ms
(do
(def process (fn [[x y]]
(+ x y)))
examples/tak.myc
+2 -1
View File
@@ -2,7 +2,8 @@
;; heavily recursive, benefits significantly from specialization of integer arithmetic
;; and direct function calls (skipping dynamic dispatch).
;; Output: 5
;; Benchmark: 461.4us
;; Benchmark-Repeat: 6
(do
examples/tuples.myc
+2 -1
View File
@@ -1,4 +1,5 @@
;; Benchmark: 1.9us
;; Benchmark: 1.7us
;; Benchmark-Repeat: 1196
;; Demonstration of N-dimensional Tuples, Vectors, and Matrices
;; Based on docs/Tupel 1.md
;;
src/utils/tester.rs
+165 -89
View File
@@ -68,103 +68,179 @@ pub fn run_benchmarks(update: bool) -> Vec<BenchmarkResult> {
let entries = fs::read_dir("examples").unwrap();
let is_release = !cfg!(debug_assertions);
let baseline_re = Regex::new(r";; Benchmark: ([\d\.]+\w+)").unwrap();
let repeat_re = Regex::new(r";; Benchmark-Repeat: (\d+)").unwrap();
for entry in entries.filter_map(|e| e.ok()) {
let env = Environment::new(); // Fresh environment per benchmark for isolation
let path = entry.path();
if path.extension().is_some_and(|ext| ext == "myc") {
let content = fs::read_to_string(&path).unwrap();
let name = path.file_name().unwrap().to_string_lossy().to_string();
if !path.extension().is_some_and(|ext| ext == "myc") {
continue;
}
let baseline_match = baseline_re.captures(&content);
let content = fs::read_to_string(&path).unwrap();
let name = path.file_name().unwrap().to_string_lossy().to_string();
// Prepare: Compile and Link once outside the measurement loop
let linked_node = match env.compile(&content).map(|c| env.link(c)) {
Ok(node) => node,
Err(e) => {
results.push(BenchmarkResult {
name,
median: Duration::ZERO,
baseline: None,
diff_pct: None,
status: format!("COMPILE ERROR: {}", e),
});
continue;
let baseline_match = baseline_re.captures(&content);
let repeat_match = repeat_re.captures(&content);
let mut repeats = repeat_match
.and_then(|m| m.get(1))
.and_then(|m| m.as_str().parse::<u32>().ok())
.unwrap_or(1);
// Compile once for this file (symbols/types are compatible with all fresh environments)
let initial_env = Environment::new();
let compiled_once = match initial_env.compile(&content) {
Ok(c) => c,
Err(e) => {
results.push(BenchmarkResult {
name,
median: Duration::ZERO,
baseline: None,
diff_pct: None,
status: format!("COMPILE ERROR: {}", e),
});
continue;
}
};
// Helper to measure sum of VM execution times over N fresh environments
let measure_sum = |n: u32, node: &crate::ast::compiler::TypedNode| -> Result<Duration, String> {
let mut total = Duration::ZERO;
for _ in 0..n {
let env = Environment::new();
// Link is still required per environment as it populates registries
let linked = env.link(node.clone());
let start = Instant::now();
let _ = env.run(&linked)?;
total += start.elapsed();
}
Ok(total)
};
if update {
repeats = 1;
loop {
match measure_sum(repeats, &compiled_once) {
Ok(total) => {
if total >= Duration::from_millis(2) || repeats >= 100_000 {
break;
}
let nanos = total.as_nanos().max(1) as f64;
let factor = 2_000_000.0 / nanos;
repeats = (repeats as f64 * factor).ceil() as u32;
repeats = repeats.max(repeats + 1);
}
Err(e) => {
results.push(BenchmarkResult {
name: name.clone(),
median: Duration::ZERO,
baseline: None,
diff_pct: None,
status: format!("ERROR: {}", e),
});
break;
}
}
}
if results.last().map_or(false, |r| r.name == name) { continue; }
}
let mut runs = Vec::new();
let mut error = None;
// Adaptive samples: High repeats need fewer samples for stable median
let num_samples = if repeats > 1000 { 10 } else if repeats > 100 { 30 } else { 100 };
for _ in 0..num_samples {
match measure_sum(repeats, &compiled_once) {
Ok(d) => runs.push(d),
Err(e) => {
error = Some(e);
break;
}
}
}
if let Some(e) = error {
results.push(BenchmarkResult {
name,
median: Duration::ZERO,
baseline: None,
diff_pct: None,
status: format!("ERROR: {}", e),
});
continue;
}
runs.sort();
let median_total = runs[runs.len() / 2];
let median_single = median_total / repeats;
if update {
let new_val = format_duration(median_single);
let mut updated_content = content.clone();
// 1. Update/Insert Benchmark
let bench_line = format!(";; Benchmark: {}", new_val);
if let Some(m) = baseline_match {
updated_content = updated_content.replace(m.get(0).unwrap().as_str(), &bench_line);
} else {
updated_content = format!("{}\n{}", bench_line, updated_content);
}
// 2. Update/Insert/Remove Benchmark-Repeat
let repeat_line = if repeats > 1 { Some(format!(";; Benchmark-Repeat: {}", repeats)) } else { None };
let current_repeat_str = repeat_re.captures(&updated_content).map(|m| m.get(0).unwrap().as_str().to_string());
if let Some(line) = repeat_line {
if let Some(old_line) = current_repeat_str {
updated_content = updated_content.replace(&old_line, &line);
} else if let Some(m) = baseline_re.captures(&updated_content) {
let b_str = m.get(0).unwrap().as_str().to_string();
if let Some(pos) = updated_content.find(&b_str) {
updated_content.insert_str(pos + b_str.len(), &format!("\n{}", line));
}
}
} else if let Some(old_line) = current_repeat_str {
updated_content = updated_content.replace(&format!("{}\n", old_line), "");
updated_content = updated_content.replace(&old_line, "");
}
fs::write(&path, updated_content).unwrap();
results.push(BenchmarkResult {
name,
median: median_single,
baseline: None,
diff_pct: None,
status: format!("UPDATED: {}", new_val),
});
} else if let Some(m) = baseline_match {
let baseline_str = m.get(1).unwrap().as_str();
let baseline = parse_duration(baseline_str).unwrap();
let diff = (median_single.as_nanos() as f64 / baseline.as_nanos() as f64) - 1.0;
let threshold = if is_release { 0.15 } else { 0.5 };
let status = if median_single > baseline && diff > threshold {
"FAILED"
} else {
"OK"
};
// Measure: Only the VM execution
let mut runs = Vec::new();
for _ in 0..100 {
let start = Instant::now();
let _ = env.run(&linked_node);
runs.push(start.elapsed());
}
runs.sort();
let median = runs[runs.len() / 2];
if update {
let new_val = format_duration(median);
let updated_content = if let Some(m) = baseline_match {
content.replace(
m.get(0).unwrap().as_str(),
&format!(";; Benchmark: {}", new_val),
)
} else {
format!(
";; Benchmark: {}
{}",
new_val, content
)
};
fs::write(&path, updated_content).unwrap();
results.push(BenchmarkResult {
name,
median,
baseline: None,
diff_pct: None,
status: format!("UPDATED: {}", new_val),
});
} else if let Some(m) = baseline_match {
let baseline_str = m.get(1).unwrap().as_str();
let baseline = parse_duration(baseline_str).unwrap();
if baseline < Duration::from_micros(1) {
results.push(BenchmarkResult {
name,
median,
baseline: Some(baseline),
diff_pct: None,
status: "SKIPPED".to_string(),
});
continue;
}
let diff = (median.as_nanos() as f64 / baseline.as_nanos() as f64) - 1.0;
let threshold = if is_release { 0.15 } else { 0.5 };
let status = if median > baseline && diff > threshold {
"FAILED"
} else {
"OK"
};
results.push(BenchmarkResult {
name,
median,
baseline: Some(baseline),
diff_pct: Some(diff * 100.0),
status: status.to_string(),
});
} else {
results.push(BenchmarkResult {
name,
median,
baseline: None,
diff_pct: None,
status: "MISSING BASELINE".to_string(),
});
}
results.push(BenchmarkResult {
name,
median: median_single,
baseline: Some(baseline),
diff_pct: Some(diff * 100.0),
status: status.to_string(),
});
} else {
results.push(BenchmarkResult {
name,
median: median_single,
baseline: None,
diff_pct: None,
status: "MISSING BASELINE".to_string(),
});
}
}
results