Update tester.rs
This commit is contained in:
+36
-21
@@ -81,7 +81,7 @@ pub fn run_benchmarks(update: bool) -> Vec<BenchmarkResult> {
|
|||||||
|
|
||||||
let baseline_match = baseline_re.captures(&content);
|
let baseline_match = baseline_re.captures(&content);
|
||||||
let repeat_match = repeat_re.captures(&content);
|
let repeat_match = repeat_re.captures(&content);
|
||||||
|
|
||||||
let mut repeats = repeat_match
|
let mut repeats = repeat_match
|
||||||
.and_then(|m| m.get(1))
|
.and_then(|m| m.get(1))
|
||||||
.and_then(|m| m.as_str().parse::<u32>().ok())
|
.and_then(|m| m.as_str().parse::<u32>().ok())
|
||||||
@@ -104,18 +104,19 @@ pub fn run_benchmarks(update: bool) -> Vec<BenchmarkResult> {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Helper to measure sum of VM execution times over N fresh environments
|
// Helper to measure sum of VM execution times over N fresh environments
|
||||||
let measure_sum = |n: u32, node: &crate::ast::compiler::TypedNode| -> Result<Duration, String> {
|
let measure_sum =
|
||||||
let mut total = Duration::ZERO;
|
|n: u32, node: &crate::ast::compiler::TypedNode| -> Result<Duration, String> {
|
||||||
for _ in 0..n {
|
let mut total = Duration::ZERO;
|
||||||
let env = Environment::new();
|
for _ in 0..n {
|
||||||
// Link is still required per environment as it populates registries
|
let env = Environment::new();
|
||||||
let linked = env.link(node.clone());
|
// Link is still required per environment as it populates registries
|
||||||
let start = Instant::now();
|
let linked = env.link(node.clone());
|
||||||
let _ = env.run(&linked)?;
|
let start = Instant::now();
|
||||||
total += start.elapsed();
|
let _ = env.run(&linked)?;
|
||||||
}
|
total += start.elapsed();
|
||||||
Ok(total)
|
}
|
||||||
};
|
Ok(total)
|
||||||
|
};
|
||||||
|
|
||||||
if update {
|
if update {
|
||||||
repeats = 1;
|
repeats = 1;
|
||||||
@@ -142,14 +143,22 @@ pub fn run_benchmarks(update: bool) -> Vec<BenchmarkResult> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if results.last().map_or(false, |r| r.name == name) { continue; }
|
if results.last().map_or(false, |r| r.name == name) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut runs = Vec::new();
|
let mut runs = Vec::new();
|
||||||
let mut error = None;
|
let mut error = None;
|
||||||
// Adaptive samples: High repeats need fewer samples for stable median
|
// Adaptive samples: High repeats need fewer samples for stable median
|
||||||
let num_samples = if repeats > 1000 { 10 } else if repeats > 100 { 30 } else { 100 };
|
let num_samples = if repeats > 1000 {
|
||||||
|
10
|
||||||
|
} else if repeats > 100 {
|
||||||
|
30
|
||||||
|
} else {
|
||||||
|
100
|
||||||
|
};
|
||||||
|
|
||||||
for _ in 0..num_samples {
|
for _ in 0..num_samples {
|
||||||
match measure_sum(repeats, &compiled_once) {
|
match measure_sum(repeats, &compiled_once) {
|
||||||
Ok(d) => runs.push(d),
|
Ok(d) => runs.push(d),
|
||||||
@@ -178,7 +187,7 @@ pub fn run_benchmarks(update: bool) -> Vec<BenchmarkResult> {
|
|||||||
if update {
|
if update {
|
||||||
let new_val = format_duration(median_single);
|
let new_val = format_duration(median_single);
|
||||||
let mut updated_content = content.clone();
|
let mut updated_content = content.clone();
|
||||||
|
|
||||||
// 1. Update/Insert Benchmark
|
// 1. Update/Insert Benchmark
|
||||||
let bench_line = format!(";; Benchmark: {}", new_val);
|
let bench_line = format!(";; Benchmark: {}", new_val);
|
||||||
if let Some(m) = baseline_match {
|
if let Some(m) = baseline_match {
|
||||||
@@ -188,8 +197,14 @@ pub fn run_benchmarks(update: bool) -> Vec<BenchmarkResult> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 2. Update/Insert/Remove Benchmark-Repeat
|
// 2. Update/Insert/Remove Benchmark-Repeat
|
||||||
let repeat_line = if repeats > 1 { Some(format!(";; Benchmark-Repeat: {}", repeats)) } else { None };
|
let repeat_line = if repeats > 1 {
|
||||||
let current_repeat_str = repeat_re.captures(&updated_content).map(|m| m.get(0).unwrap().as_str().to_string());
|
Some(format!(";; Benchmark-Repeat: {}", repeats))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let current_repeat_str = repeat_re
|
||||||
|
.captures(&updated_content)
|
||||||
|
.map(|m| m.get(0).unwrap().as_str().to_string());
|
||||||
|
|
||||||
if let Some(line) = repeat_line {
|
if let Some(line) = repeat_line {
|
||||||
if let Some(old_line) = current_repeat_str {
|
if let Some(old_line) = current_repeat_str {
|
||||||
@@ -197,7 +212,7 @@ pub fn run_benchmarks(update: bool) -> Vec<BenchmarkResult> {
|
|||||||
} else if let Some(m) = baseline_re.captures(&updated_content) {
|
} else if let Some(m) = baseline_re.captures(&updated_content) {
|
||||||
let b_str = m.get(0).unwrap().as_str().to_string();
|
let b_str = m.get(0).unwrap().as_str().to_string();
|
||||||
if let Some(pos) = updated_content.find(&b_str) {
|
if let Some(pos) = updated_content.find(&b_str) {
|
||||||
updated_content.insert_str(pos + b_str.len(), &format!("\n{}", line));
|
updated_content.insert_str(pos + b_str.len(), &format!("\n{}", line));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if let Some(old_line) = current_repeat_str {
|
} else if let Some(old_line) = current_repeat_str {
|
||||||
@@ -219,7 +234,7 @@ pub fn run_benchmarks(update: bool) -> Vec<BenchmarkResult> {
|
|||||||
|
|
||||||
let diff = (median_single.as_nanos() as f64 / baseline.as_nanos() as f64) - 1.0;
|
let diff = (median_single.as_nanos() as f64 / baseline.as_nanos() as f64) - 1.0;
|
||||||
|
|
||||||
let threshold = if is_release { 0.15 } else { 0.5 };
|
let threshold = if is_release { 0.10 } else { 0.25 };
|
||||||
let status = if median_single > baseline && diff > threshold {
|
let status = if median_single > baseline && diff > threshold {
|
||||||
"FAILED"
|
"FAILED"
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
Reference in New Issue
Block a user