diff --git a/Cargo.lock b/Cargo.lock index c94064c..607f987 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -127,6 +127,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + [[package]] name = "android-activity" version = "0.6.0" @@ -1882,6 +1891,7 @@ dependencies = [ "clap", "eframe", "lazy_static", + "regex", ] [[package]] @@ -2562,6 +2572,35 @@ dependencies = [ "bitflags 2.11.0", ] +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" + [[package]] name = "renderdoc-sys" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 468567a..7699384 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,3 +9,4 @@ eframe = "0.33.3" lazy_static = "1.4.0" clap = { version = "4.5", features = ["derive"] } chrono = "0.4" +regex = "1.10" diff --git a/examples/closure.myc b/examples/closure.myc index 98f649f..a1272f8 100644 --- a/examples/closure.myc +++ b/examples/closure.myc @@ -1,4 +1,5 @@ ;; Closure Scope Test +;; Benchmark: 7.9us ;; Output: 15 (do (def make-adder (fn [x] diff --git a/examples/data.myc b/examples/data.myc new file mode 100644 index 0000000..2c10e98 --- /dev/null +++ b/examples/data.myc @@ -0,0 +1,19 @@ +;; Complex Data Structure Test +;; Benchmark: 66.8us +;; Output: {:input 10, :name "Fibonacci", :output 55} +(do + (def fib (fn [n] + (if (< n 2) + n + (+ (fib (- n 1)) (fib (- n 2)))))) + + (def result (fib 10)) + + (def data { + :name "Fibonacci" + :input 10 + :output result + }) + + data +) diff --git a/examples/fib.myc b/examples/fib.myc index 5aa538d..904490c 100644 --- a/examples/fib.myc +++ b/examples/fib.myc @@ -1,4 +1,5 @@ ;; Fibonacci Recursive +;; Benchmark: 62.9us ;; Output: 55 (do (def fib (fn [n] diff --git a/examples/hof.myc b/examples/hof.myc index fd70109..b0390ef 100644 --- a/examples/hof.myc +++ b/examples/hof.myc @@ -1,4 +1,5 @@ ;; Higher-Order Function Example +;; Benchmark: 7.4us ;; Output: 25 (do (def apply (fn [f x] (f x))) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 3f46f68..0381911 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -1,15 +1,8 @@ pub mod types; -pub mod nodes; pub mod lexer; +pub mod nodes; pub mod parser; pub mod compiler; -pub mod vm; pub mod environment; - -pub use types::*; -pub use nodes::*; -pub use lexer::*; -pub use parser::*; -pub use compiler::*; -pub use vm::*; -pub use environment::*; +pub mod vm; +pub mod tester; diff --git a/src/ast/tester.rs b/src/ast/tester.rs new file mode 100644 index 0000000..879b434 --- /dev/null +++ b/src/ast/tester.rs @@ -0,0 +1,126 @@ +use crate::ast::environment::Environment; +use std::fs; +use std::time::{Duration, Instant}; +use regex::Regex; + +pub struct TestResult { + pub name: String, + pub success: bool, + pub message: String, +} + +pub struct BenchmarkResult { + pub name: String, + pub median: Duration, + pub baseline: Option, + pub diff_pct: Option, + pub status: String, +} + +pub fn run_functional_tests() -> Vec { + let mut results = Vec::new(); + let entries = fs::read_dir("examples").unwrap(); + let env = Environment::new(); + + for entry in entries.filter_map(|e| e.ok()) { + let path = entry.path(); + if path.extension().map_or(false, |ext| ext == "myc") { + let content = fs::read_to_string(&path).unwrap(); + let name = path.file_name().unwrap().to_string_lossy().to_string(); + + let expected_output = Regex::new(r";; Output: (.*)").unwrap() + .captures(&content) + .map(|m| m.get(1).unwrap().as_str().trim().to_string()); + + if let Some(expected) = expected_output { + match env.run_script(&content) { + Ok(val) => { + let val_str = format!("{}", val); + if val_str == expected { + results.push(TestResult { name, success: true, message: format!("OK: {}", val_str) }); + } else { + results.push(TestResult { name, success: false, message: format!("Expected {}, got {}", expected, val_str) }); + } + } + Err(e) => results.push(TestResult { name, success: false, message: format!("Error: {}", e) }), + } + } + } + } + results +} + +pub fn run_benchmarks(update: bool) -> Vec { + let mut results = Vec::new(); + let entries = fs::read_dir("examples").unwrap(); + let env = Environment::new(); + let is_release = !cfg!(debug_assertions); + + for entry in entries.filter_map(|e| e.ok()) { + let path = entry.path(); + if path.extension().map_or(false, |ext| ext == "myc") { + let content = fs::read_to_string(&path).unwrap(); + let name = path.file_name().unwrap().to_string_lossy().to_string(); + + let baseline_pattern = Regex::new(r";; Benchmark: ([\d\.]+\w+)").unwrap(); + let baseline_match = baseline_pattern.captures(&content); + + // Measure + let mut runs = Vec::new(); + for _ in 0..100 { + let start = Instant::now(); + let _ = env.run_script(&content); + runs.push(start.elapsed()); + } + runs.sort(); + let median = runs[runs.len() / 2]; + + if update { + let new_val = format_duration(median); + let updated_content = if let Some(m) = baseline_match { + content.replace(m.get(0).unwrap().as_str(), &format!(";; Benchmark: {}", new_val)) + } else { + format!(";; Benchmark: {} +{}", new_val, content) + }; + fs::write(&path, updated_content).unwrap(); + results.push(BenchmarkResult { name, median, baseline: None, diff_pct: None, status: format!("UPDATED: {}", new_val) }); + } else { + if let Some(m) = baseline_match { + let baseline_str = m.get(1).unwrap().as_str(); + let baseline = parse_duration(baseline_str).unwrap(); + let diff = (median.as_nanos() as f64 / baseline.as_nanos() as f64) - 1.0; + + let threshold = if is_release { 0.10 } else { 0.25 }; + let status = if median > baseline && diff > threshold { "FAILED" } else { "OK" }; + + results.push(BenchmarkResult { name, median, baseline: Some(baseline), diff_pct: Some(diff * 100.0), status: status.to_string() }); + } else { + results.push(BenchmarkResult { name, median, baseline: None, diff_pct: None, status: "MISSING BASELINE".to_string() }); + } + } + } + } + results +} + +fn format_duration(d: Duration) -> String { + if d.as_nanos() < 1000 { format!("{}ns", d.as_nanos()) } + else if d.as_micros() < 1000 { format!("{:.1}us", d.as_nanos() as f64 / 1000.0) } + else if d.as_millis() < 1000 { format!("{:.1}ms", d.as_micros() as f64 / 1000.0) } + else { format!("{:.1}s", d.as_millis() as f64 / 1000.0) } +} + +fn parse_duration(s: &str) -> Option { + let re = Regex::new(r"([\d\.]+)(\w+)").unwrap(); + let caps = re.captures(s)?; + let val: f64 = caps[1].parse().ok()?; + let unit = &caps[2]; + match unit { + "ns" => Some(Duration::from_nanos(val as u64)), + "us" => Some(Duration::from_nanos((val * 1000.0) as u64)), + "ms" => Some(Duration::from_nanos((val * 1_000_000.0) as u64)), + "s" => Some(Duration::from_nanos((val * 1_000_000_000.0) as u64)), + _ => None, + } +} diff --git a/src/ast/types.rs b/src/ast/types.rs index 601d191..340b1b6 100644 --- a/src/ast/types.rs +++ b/src/ast/types.rs @@ -135,8 +135,10 @@ impl fmt::Display for Value { write!(f, "]") }, Value::Record(r) => { + let mut entries: Vec<_> = r.iter().collect(); + entries.sort_by_key(|(k, _)| k.name()); write!(f, "{{")?; - for (i, (k, v)) in r.iter().enumerate() { + for (i, (k, v)) in entries.iter().enumerate() { if i > 0 { write!(f, ", ")?; } write!(f, ":{} {}", k.name(), v)?; } diff --git a/src/bin/ast.rs b/src/bin/ast.rs index f6d9c2e..042ab10 100644 --- a/src/bin/ast.rs +++ b/src/bin/ast.rs @@ -1,24 +1,49 @@ use myc::ast::environment::Environment; +use myc::ast::tester; use clap::Parser; use std::fs; use std::path::PathBuf; #[derive(Parser)] -#[command(author, version, about = "MYC AST Compiler CLI", long_about = None)] +#[command(author, version, about = "MYC AST Compiler & Benchmarker", long_about = None)] struct Cli { - /// The script file to run + /// The script file to run or benchmark #[arg(value_name = "FILE")] file: Option, /// Run a script string directly #[arg(short, long)] eval: Option, + + /// Run benchmarks (Only allowed in Release mode) + #[arg(short, long)] + bench: bool, + + /// Update the benchmark baseline (Only allowed in Release mode) + #[arg(short, long)] + update_bench: bool, } fn main() { let cli = Cli::parse(); let env = Environment::new(); + if cli.bench || cli.update_bench { + if cfg!(debug_assertions) { + eprintln!("❌ ERROR: Benchmarks must be run in Release mode!"); + eprintln!(" Use: cargo run --release --bin ast -- --bench"); + std::process::exit(1); + } + + println!("🚀 Running benchmarks in RELEASE mode...\n"); + let results = tester::run_benchmarks(cli.update_bench); + for res in results { + let diff = res.diff_pct.map_or(String::new(), |d| format!(" ({:+.1}%)", d)); + println!("{}: {} - {:?}{}", res.status, res.name, res.median, diff); + } + return; + } + if let Some(script_str) = cli.eval { execute(&env, &script_str); } else if let Some(file_path) = cli.file { diff --git a/src/integration_test.rs b/src/integration_test.rs index 6b085b1..1891b3a 100644 --- a/src/integration_test.rs +++ b/src/integration_test.rs @@ -100,11 +100,6 @@ mod tests { if path.extension().map_or(false, |ext| ext == "myc") { let content = fs::read_to_string(&path).expect("Could not read file"); - // Skip benchmarks during normal functional tests - if content.contains(";; Benchmark") { - continue; - } - // Find expected output tag: ;; Output: let expected_output = content.lines() .find(|line| line.starts_with(";; Output:")) diff --git a/src/main.rs b/src/main.rs index 636baaa..4cf3b5e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,7 +16,6 @@ fn main() -> eframe::Result { struct Example { name: String, content: String, - is_benchmark: bool, } struct CompilerApp { @@ -37,8 +36,7 @@ impl Default for CompilerApp { .filter_map(|e| { let name = e.file_name().into_string().ok()?; let content = std::fs::read_to_string(e.path()).ok()?; - let is_benchmark = content.contains(";; Benchmark"); - Some(Example { name, content, is_benchmark }) + Some(Example { name, content }) }) .collect() }) @@ -82,6 +80,37 @@ impl CompilerApp { } } } + + fn run_all_tests(&mut self) { + use myc::ast::tester; + let results = tester::run_functional_tests(); + let mut log = String::from("FUNCTIONAL TEST RESULTS:\n\n"); + let mut passed = 0; + for res in &results { + log.push_str(&format!("{}: {} - {}\n", if res.success { "✅" } else { "❌" }, res.name, res.message)); + if res.success { passed += 1; } + } + log.push_str(&format!("\nTotal: {}/{}", passed, results.len())); + self.output_log = log; + } + + fn run_benchmarks(&mut self, update: bool) { + use myc::ast::tester; + if cfg!(debug_assertions) && !update { + self.output_log = String::from("⚠️ WARNING: Benchmarks in Debug mode are inaccurate!\nUse Release build for real measurements.\n\n"); + } else { + self.output_log = String::from(if update { "UPDATING BASELINES...\n\n" } else { "RUNNING BENCHMARKS...\n\n" }); + } + + let results = tester::run_benchmarks(update); + let mut log = self.output_log.clone(); + + for res in results { + let diff = res.diff_pct.map_or(String::new(), |d| format!(" ({:+.1}%)", d)); + log.push_str(&format!("{}: {} - {:?}{}\n", res.status, res.name, res.median, diff)); + } + self.output_log = log; + } } impl eframe::App for CompilerApp { @@ -94,12 +123,7 @@ impl eframe::App for CompilerApp { ui.heading("Examples"); ui.add_space(5.0); for example in &self.examples { - let label = if example.is_benchmark { - format!("⚡ {}", example.name) - } else { - example.name.clone() - }; - if ui.button(label).clicked() { + if ui.button(&example.name).clicked() { self.source_code = example.content.clone(); } } @@ -114,9 +138,35 @@ impl eframe::App for CompilerApp { ui.add_space(5.0); // Compile Button (at the top of the bottom panel) - if ui.button("Compile").clicked() { - self.compile(); - } + ui.horizontal(|ui| { + if ui.button("Compile").clicked() { + self.compile(); + } + + if ui.button("Test All").clicked() { + self.run_all_tests(); + } + + let is_debug = cfg!(debug_assertions); + + ui.add_enabled_ui(!is_debug, |ui| { + let btn = ui.button("Run Benchmarks"); + if is_debug { + btn.on_hover_text("Benchmarks are only available in Release builds for accuracy."); + } else if btn.clicked() { + self.run_benchmarks(false); + } + }); + + ui.add_enabled_ui(!is_debug, |ui| { + let btn = ui.button("Update Baselines"); + if is_debug { + btn.on_hover_text("Updating baselines is only allowed in Release builds."); + } else if btn.clicked() { + self.run_benchmarks(true); + } + }); + }); ui.add_space(5.0); ui.horizontal(|ui| {