Update benchmark timings and repeats
This commit updates the benchmark timings and repeat counts for various example files. The `tester.rs` script has been modified to: - Correctly parse and use the `Benchmark-Repeat` directive. - Implement an adaptive sampling mechanism for more accurate median calculation, especially for long-running benchmarks. - Improve the logic for updating and inserting benchmark and repeat lines in example files. - Handle potential compilation and runtime errors during benchmark execution.
This commit is contained in:
@@ -1,5 +1,6 @@
|
|||||||
;; Closure Scope Test
|
;; Closure Scope Test
|
||||||
;; Benchmark: 800ns
|
;; Benchmark: 780ns
|
||||||
|
;; Benchmark-Repeat: 2602
|
||||||
;; Output: 15
|
;; Output: 15
|
||||||
(do
|
(do
|
||||||
(def make-adder (fn [x]
|
(def make-adder (fn [x]
|
||||||
|
|||||||
+2
-1
@@ -1,5 +1,6 @@
|
|||||||
;; Complex Data Structure Test
|
;; Complex Data Structure Test
|
||||||
;; Benchmark: 51.4us
|
;; Benchmark: 52.9us
|
||||||
|
;; Benchmark-Repeat: 39
|
||||||
;; Output: {:name "Fibonacci", :input 10, :output 55}
|
;; Output: {:name "Fibonacci", :input 10, :output 55}
|
||||||
(do
|
(do
|
||||||
(def fib (fn [n]
|
(def fib (fn [n]
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
;; Benchmark: 2.6us
|
||||||
|
;; Benchmark-Repeat: 788
|
||||||
;; Comprehensive Destructuring Test
|
;; Comprehensive Destructuring Test
|
||||||
;; Covers: Nested tuples, mixed params, dynamic passing
|
;; Covers: Nested tuples, mixed params, dynamic passing
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
;; Benchmark: 1.6us
|
;; Benchmark: 1.8us
|
||||||
|
;; Benchmark-Repeat: 1103
|
||||||
;; Output: 36
|
;; Output: 36
|
||||||
(do
|
(do
|
||||||
;; Excessive capture test
|
;; Excessive capture test
|
||||||
|
|||||||
+2
-1
@@ -1,5 +1,6 @@
|
|||||||
;; Fibonacci Recursive
|
;; Fibonacci Recursive
|
||||||
;; Benchmark: 49.7us
|
;; Benchmark: 52.7us
|
||||||
|
;; Benchmark-Repeat: 38
|
||||||
;; Output: 55
|
;; Output: 55
|
||||||
(do
|
(do
|
||||||
(def fib (fn [n]
|
(def fib (fn [n]
|
||||||
|
|||||||
+2
-1
@@ -1,5 +1,6 @@
|
|||||||
;; Higher-Order Function Example
|
;; Higher-Order Function Example
|
||||||
;; Benchmark: 600ns
|
;; Benchmark: 725ns
|
||||||
|
;; Benchmark-Repeat: 2768
|
||||||
;; Output: 25
|
;; Output: 25
|
||||||
(do
|
(do
|
||||||
(def apply (fn [f x] (f x)))
|
(def apply (fn [f x] (f x)))
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
;; Benchmark: 400ns
|
;; Benchmark: 484ns
|
||||||
|
;; Benchmark-Repeat: 4144
|
||||||
;; Financial DSL Macro example
|
;; Financial DSL Macro example
|
||||||
;; Demonstrates generating complex records from simple parameters.
|
;; Demonstrates generating complex records from simple parameters.
|
||||||
;; Output: {:price 100, :size 2, :total 200}
|
;; Output: {:price 100, :size 2, :total 200}
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
;; Benchmark: 400ns
|
;; Benchmark: 448ns
|
||||||
|
;; Benchmark-Repeat: 4488
|
||||||
;; Output: 5
|
;; Output: 5
|
||||||
(do
|
(do
|
||||||
(def y 1)
|
(def y 1)
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
;; Benchmark: 300ns
|
;; Benchmark: 429ns
|
||||||
|
;; Benchmark-Repeat: 4712
|
||||||
;; Nested Macro and Arithmetic example
|
;; Nested Macro and Arithmetic example
|
||||||
;; Output: 81
|
;; Output: 81
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
;; Benchmark: 300ns
|
;; Benchmark: 320ns
|
||||||
|
;; Benchmark-Repeat: 6238
|
||||||
;; Macro Splicing example
|
;; Macro Splicing example
|
||||||
;; Demonstrates unrolling a list into another list.
|
;; Demonstrates unrolling a list into another list.
|
||||||
;; Output: [0 1 2 3 4]
|
;; Output: [0 1 2 3 4]
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
;; Benchmark: 200ns
|
;; Benchmark: 209ns
|
||||||
|
;; Benchmark-Repeat: 9540
|
||||||
;; Classic "unless" macro example
|
;; Classic "unless" macro example
|
||||||
;; Demonstrates simple template substitution.
|
;; Demonstrates simple template substitution.
|
||||||
;; Output: 42
|
;; Output: 42
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
;; Record to Tuple Mapping Test
|
;; Record to Tuple Mapping Test
|
||||||
;; Output: 30
|
;; Output: 30
|
||||||
;; Benchmark: 700ns
|
;; Benchmark: 755ns
|
||||||
|
;; Benchmark-Repeat: 2652
|
||||||
(do
|
(do
|
||||||
(def rec {:x 10 :y 20})
|
(def rec {:x 10 :y 20})
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,8 @@
|
|||||||
;; This test calls a function that destructures a record 100.000 times.
|
;; This test calls a function that destructures a record 100.000 times.
|
||||||
;; Current implementation is now zero-allocation for destructuring.
|
;; Current implementation is now zero-allocation for destructuring.
|
||||||
;; Output: {:a 1, :b 2}
|
;; Output: {:a 1, :b 2}
|
||||||
;; Benchmark: 5.3ms
|
;; Benchmark: 5.2ms
|
||||||
|
|
||||||
(do
|
(do
|
||||||
(def process (fn [[x y]]
|
(def process (fn [[x y]]
|
||||||
(+ x y)))
|
(+ x y)))
|
||||||
|
|||||||
+2
-1
@@ -2,7 +2,8 @@
|
|||||||
;; heavily recursive, benefits significantly from specialization of integer arithmetic
|
;; heavily recursive, benefits significantly from specialization of integer arithmetic
|
||||||
;; and direct function calls (skipping dynamic dispatch).
|
;; and direct function calls (skipping dynamic dispatch).
|
||||||
;; Output: 5
|
;; Output: 5
|
||||||
|
|
||||||
|
;; Benchmark: 461.4us
|
||||||
;; Benchmark-Repeat: 6
|
;; Benchmark-Repeat: 6
|
||||||
|
|
||||||
(do
|
(do
|
||||||
|
|||||||
+2
-1
@@ -1,4 +1,5 @@
|
|||||||
;; Benchmark: 1.9us
|
;; Benchmark: 1.7us
|
||||||
|
;; Benchmark-Repeat: 1196
|
||||||
;; Demonstration of N-dimensional Tuples, Vectors, and Matrices
|
;; Demonstration of N-dimensional Tuples, Vectors, and Matrices
|
||||||
;; Based on docs/Tupel 1.md
|
;; Based on docs/Tupel 1.md
|
||||||
;;
|
;;
|
||||||
|
|||||||
+165
-89
@@ -68,103 +68,179 @@ pub fn run_benchmarks(update: bool) -> Vec<BenchmarkResult> {
|
|||||||
let entries = fs::read_dir("examples").unwrap();
|
let entries = fs::read_dir("examples").unwrap();
|
||||||
let is_release = !cfg!(debug_assertions);
|
let is_release = !cfg!(debug_assertions);
|
||||||
let baseline_re = Regex::new(r";; Benchmark: ([\d\.]+\w+)").unwrap();
|
let baseline_re = Regex::new(r";; Benchmark: ([\d\.]+\w+)").unwrap();
|
||||||
|
let repeat_re = Regex::new(r";; Benchmark-Repeat: (\d+)").unwrap();
|
||||||
|
|
||||||
for entry in entries.filter_map(|e| e.ok()) {
|
for entry in entries.filter_map(|e| e.ok()) {
|
||||||
let env = Environment::new(); // Fresh environment per benchmark for isolation
|
|
||||||
let path = entry.path();
|
let path = entry.path();
|
||||||
if path.extension().is_some_and(|ext| ext == "myc") {
|
if !path.extension().is_some_and(|ext| ext == "myc") {
|
||||||
let content = fs::read_to_string(&path).unwrap();
|
continue;
|
||||||
let name = path.file_name().unwrap().to_string_lossy().to_string();
|
}
|
||||||
|
|
||||||
let baseline_match = baseline_re.captures(&content);
|
let content = fs::read_to_string(&path).unwrap();
|
||||||
|
let name = path.file_name().unwrap().to_string_lossy().to_string();
|
||||||
|
|
||||||
// Prepare: Compile and Link once outside the measurement loop
|
let baseline_match = baseline_re.captures(&content);
|
||||||
let linked_node = match env.compile(&content).map(|c| env.link(c)) {
|
let repeat_match = repeat_re.captures(&content);
|
||||||
Ok(node) => node,
|
|
||||||
Err(e) => {
|
let mut repeats = repeat_match
|
||||||
results.push(BenchmarkResult {
|
.and_then(|m| m.get(1))
|
||||||
name,
|
.and_then(|m| m.as_str().parse::<u32>().ok())
|
||||||
median: Duration::ZERO,
|
.unwrap_or(1);
|
||||||
baseline: None,
|
|
||||||
diff_pct: None,
|
// Compile once for this file (symbols/types are compatible with all fresh environments)
|
||||||
status: format!("COMPILE ERROR: {}", e),
|
let initial_env = Environment::new();
|
||||||
});
|
let compiled_once = match initial_env.compile(&content) {
|
||||||
continue;
|
Ok(c) => c,
|
||||||
|
Err(e) => {
|
||||||
|
results.push(BenchmarkResult {
|
||||||
|
name,
|
||||||
|
median: Duration::ZERO,
|
||||||
|
baseline: None,
|
||||||
|
diff_pct: None,
|
||||||
|
status: format!("COMPILE ERROR: {}", e),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Helper to measure sum of VM execution times over N fresh environments
|
||||||
|
let measure_sum = |n: u32, node: &crate::ast::compiler::TypedNode| -> Result<Duration, String> {
|
||||||
|
let mut total = Duration::ZERO;
|
||||||
|
for _ in 0..n {
|
||||||
|
let env = Environment::new();
|
||||||
|
// Link is still required per environment as it populates registries
|
||||||
|
let linked = env.link(node.clone());
|
||||||
|
let start = Instant::now();
|
||||||
|
let _ = env.run(&linked)?;
|
||||||
|
total += start.elapsed();
|
||||||
|
}
|
||||||
|
Ok(total)
|
||||||
|
};
|
||||||
|
|
||||||
|
if update {
|
||||||
|
repeats = 1;
|
||||||
|
loop {
|
||||||
|
match measure_sum(repeats, &compiled_once) {
|
||||||
|
Ok(total) => {
|
||||||
|
if total >= Duration::from_millis(2) || repeats >= 100_000 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let nanos = total.as_nanos().max(1) as f64;
|
||||||
|
let factor = 2_000_000.0 / nanos;
|
||||||
|
repeats = (repeats as f64 * factor).ceil() as u32;
|
||||||
|
repeats = repeats.max(repeats + 1);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
results.push(BenchmarkResult {
|
||||||
|
name: name.clone(),
|
||||||
|
median: Duration::ZERO,
|
||||||
|
baseline: None,
|
||||||
|
diff_pct: None,
|
||||||
|
status: format!("ERROR: {}", e),
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
if results.last().map_or(false, |r| r.name == name) { continue; }
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut runs = Vec::new();
|
||||||
|
let mut error = None;
|
||||||
|
// Adaptive samples: High repeats need fewer samples for stable median
|
||||||
|
let num_samples = if repeats > 1000 { 10 } else if repeats > 100 { 30 } else { 100 };
|
||||||
|
|
||||||
|
for _ in 0..num_samples {
|
||||||
|
match measure_sum(repeats, &compiled_once) {
|
||||||
|
Ok(d) => runs.push(d),
|
||||||
|
Err(e) => {
|
||||||
|
error = Some(e);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(e) = error {
|
||||||
|
results.push(BenchmarkResult {
|
||||||
|
name,
|
||||||
|
median: Duration::ZERO,
|
||||||
|
baseline: None,
|
||||||
|
diff_pct: None,
|
||||||
|
status: format!("ERROR: {}", e),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
runs.sort();
|
||||||
|
let median_total = runs[runs.len() / 2];
|
||||||
|
let median_single = median_total / repeats;
|
||||||
|
|
||||||
|
if update {
|
||||||
|
let new_val = format_duration(median_single);
|
||||||
|
let mut updated_content = content.clone();
|
||||||
|
|
||||||
|
// 1. Update/Insert Benchmark
|
||||||
|
let bench_line = format!(";; Benchmark: {}", new_val);
|
||||||
|
if let Some(m) = baseline_match {
|
||||||
|
updated_content = updated_content.replace(m.get(0).unwrap().as_str(), &bench_line);
|
||||||
|
} else {
|
||||||
|
updated_content = format!("{}\n{}", bench_line, updated_content);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Update/Insert/Remove Benchmark-Repeat
|
||||||
|
let repeat_line = if repeats > 1 { Some(format!(";; Benchmark-Repeat: {}", repeats)) } else { None };
|
||||||
|
let current_repeat_str = repeat_re.captures(&updated_content).map(|m| m.get(0).unwrap().as_str().to_string());
|
||||||
|
|
||||||
|
if let Some(line) = repeat_line {
|
||||||
|
if let Some(old_line) = current_repeat_str {
|
||||||
|
updated_content = updated_content.replace(&old_line, &line);
|
||||||
|
} else if let Some(m) = baseline_re.captures(&updated_content) {
|
||||||
|
let b_str = m.get(0).unwrap().as_str().to_string();
|
||||||
|
if let Some(pos) = updated_content.find(&b_str) {
|
||||||
|
updated_content.insert_str(pos + b_str.len(), &format!("\n{}", line));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if let Some(old_line) = current_repeat_str {
|
||||||
|
updated_content = updated_content.replace(&format!("{}\n", old_line), "");
|
||||||
|
updated_content = updated_content.replace(&old_line, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
fs::write(&path, updated_content).unwrap();
|
||||||
|
results.push(BenchmarkResult {
|
||||||
|
name,
|
||||||
|
median: median_single,
|
||||||
|
baseline: None,
|
||||||
|
diff_pct: None,
|
||||||
|
status: format!("UPDATED: {}", new_val),
|
||||||
|
});
|
||||||
|
} else if let Some(m) = baseline_match {
|
||||||
|
let baseline_str = m.get(1).unwrap().as_str();
|
||||||
|
let baseline = parse_duration(baseline_str).unwrap();
|
||||||
|
|
||||||
|
let diff = (median_single.as_nanos() as f64 / baseline.as_nanos() as f64) - 1.0;
|
||||||
|
|
||||||
|
let threshold = if is_release { 0.15 } else { 0.5 };
|
||||||
|
let status = if median_single > baseline && diff > threshold {
|
||||||
|
"FAILED"
|
||||||
|
} else {
|
||||||
|
"OK"
|
||||||
};
|
};
|
||||||
|
|
||||||
// Measure: Only the VM execution
|
results.push(BenchmarkResult {
|
||||||
let mut runs = Vec::new();
|
name,
|
||||||
for _ in 0..100 {
|
median: median_single,
|
||||||
let start = Instant::now();
|
baseline: Some(baseline),
|
||||||
let _ = env.run(&linked_node);
|
diff_pct: Some(diff * 100.0),
|
||||||
runs.push(start.elapsed());
|
status: status.to_string(),
|
||||||
}
|
});
|
||||||
runs.sort();
|
} else {
|
||||||
let median = runs[runs.len() / 2];
|
results.push(BenchmarkResult {
|
||||||
|
name,
|
||||||
if update {
|
median: median_single,
|
||||||
let new_val = format_duration(median);
|
baseline: None,
|
||||||
let updated_content = if let Some(m) = baseline_match {
|
diff_pct: None,
|
||||||
content.replace(
|
status: "MISSING BASELINE".to_string(),
|
||||||
m.get(0).unwrap().as_str(),
|
});
|
||||||
&format!(";; Benchmark: {}", new_val),
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
format!(
|
|
||||||
";; Benchmark: {}
|
|
||||||
{}",
|
|
||||||
new_val, content
|
|
||||||
)
|
|
||||||
};
|
|
||||||
fs::write(&path, updated_content).unwrap();
|
|
||||||
results.push(BenchmarkResult {
|
|
||||||
name,
|
|
||||||
median,
|
|
||||||
baseline: None,
|
|
||||||
diff_pct: None,
|
|
||||||
status: format!("UPDATED: {}", new_val),
|
|
||||||
});
|
|
||||||
} else if let Some(m) = baseline_match {
|
|
||||||
let baseline_str = m.get(1).unwrap().as_str();
|
|
||||||
let baseline = parse_duration(baseline_str).unwrap();
|
|
||||||
|
|
||||||
if baseline < Duration::from_micros(1) {
|
|
||||||
results.push(BenchmarkResult {
|
|
||||||
name,
|
|
||||||
median,
|
|
||||||
baseline: Some(baseline),
|
|
||||||
diff_pct: None,
|
|
||||||
status: "SKIPPED".to_string(),
|
|
||||||
});
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
let diff = (median.as_nanos() as f64 / baseline.as_nanos() as f64) - 1.0;
|
|
||||||
|
|
||||||
let threshold = if is_release { 0.15 } else { 0.5 };
|
|
||||||
let status = if median > baseline && diff > threshold {
|
|
||||||
"FAILED"
|
|
||||||
} else {
|
|
||||||
"OK"
|
|
||||||
};
|
|
||||||
|
|
||||||
results.push(BenchmarkResult {
|
|
||||||
name,
|
|
||||||
median,
|
|
||||||
baseline: Some(baseline),
|
|
||||||
diff_pct: Some(diff * 100.0),
|
|
||||||
status: status.to_string(),
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
results.push(BenchmarkResult {
|
|
||||||
name,
|
|
||||||
median,
|
|
||||||
baseline: None,
|
|
||||||
diff_pct: None,
|
|
||||||
status: "MISSING BASELINE".to_string(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
results
|
results
|
||||||
|
|||||||
Reference in New Issue
Block a user