From 94fc6bf56d9eded7125ca49fb61cfb98cad59bc6 Mon Sep 17 00:00:00 2001 From: Michael Schimmel Date: Wed, 18 Feb 2026 12:51:07 +0100 Subject: [PATCH] feat: Implement AST macros and templates This commit introduces support for AST macros and templates, enabling users to define and use custom, reusable components within the visual DSL. Key changes include: - A new `MacroRegistry` to manage macro definitions. - A `MacroExpander` to process macro calls and expand templates. - A `MacroEvaluator` trait for evaluating expressions during expansion. - The `Expansion` node in `BoundKind` to preserve the original macro call and its expanded form for debugging. - Updates to the `Binder`, `Dumper`, and `UpvalueAnalyzer` to handle the new macro constructs. - New examples demonstrating various macro functionalities like `unless`, splicing, and nested macros. --- docs/macros_and_templates.md | 48 ++++ examples/closure.myc | 2 +- examples/data.myc | 2 +- examples/extreme_capture.myc | 2 +- examples/fib.myc | 2 +- examples/hof.myc | 2 +- examples/macro_financial.myc | 8 + examples/macro_power.myc | 8 + examples/macro_splice.myc | 8 + examples/macro_unless.myc | 8 + src/ast/compiler/binder.rs | 13 + src/ast/compiler/bound_nodes.rs | 8 + src/ast/compiler/dumper.rs | 7 + src/ast/compiler/macros.rs | 453 ++++++++++++++++++++++++++++++++ src/ast/compiler/mod.rs | 2 + src/ast/compiler/upvalues.rs | 6 + src/ast/environment.rs | 109 +++++--- src/ast/lexer.rs | 19 +- src/ast/nodes.rs | 40 ++- src/ast/parser.rs | 48 +++- src/ast/tester.rs | 10 +- src/ast/vm.rs | 4 + src/main.rs | 56 ++-- 23 files changed, 798 insertions(+), 67 deletions(-) create mode 100644 docs/macros_and_templates.md create mode 100644 examples/macro_financial.myc create mode 100644 examples/macro_power.myc create mode 100644 examples/macro_splice.myc create mode 100644 examples/macro_unless.myc create mode 100644 src/ast/compiler/macros.rs diff --git a/docs/macros_and_templates.md b/docs/macros_and_templates.md new file mode 100644 index 0000000..6760ed9 --- /dev/null +++ b/docs/macros_and_templates.md @@ -0,0 +1,48 @@ +# AST Macros & Templates (Portierung von Delphi nach Rust) + +Dieses Dokument beschreibt die Architektur und Nomenklatur für das Makro- und Template-System im Rust-AST-Compiler. + +## 1. Motivation: Warum Makros in einer visuellen DSL? + +In einer visuellen DSL dienen Makros als **Benutzerdefinierte Komponenten**. Ein User baut einen Graphen (den Body), definiert Eingänge (Placeholder) und deklariert dieses Template unter einem Namen. Bei der Kompilierung wird dieses Template an den Aufrufstellen "expandiert". + +**Vorteile:** +- **Abstraktion:** Komplexe Logik wird hinter einem einzigen Block in der UI versteckt. +- **Wiederverwendbarkeit:** Einmal deklarierte Finanz-Indikatoren können überall eingesetzt werden. +- **Kompilierzeit-Optimierung:** Strukturelle Transformationen (z.B. Loop-Unrolling) können über Makros abgebildet werden. + +## 2. Nomenklatur (Visuelle Ebene statt Lisp-Ebene) + +Um die Kluft zwischen Text-Logik (Lisp) und visueller Logik zu schließen, wurde folgende Nomenklatur gewählt: + +| Konzept | Name in Rust | Funktion | +| :--- | :--- | :--- | +| `akMacroDefinition` | `MacroDecl` | Deklariert ein benanntes Template als Compiler-Erweiterung. | +| `akQuasiquote` | `Template` | Ein Rahmen (Sub-Graph), in dem Placeholder ersetzt werden können. | +| `akUnquote` | `Placeholder` | Ein "Loch" oder "Input-Pin" innerhalb eines Templates. | +| `akUnquoteSplicing` | `Splice` | Ein spezialisierter Placeholder, der Inhalte "auspackt" und verschmilzt. | +| `akMacroExpansion` | `Expansion` | Ein Hybrid-Knoten, der den Original-Aufruf und das Ergebnis speichert. | + +## 3. Semantik von "Splice" (Die Verschmelzung) + +Ein `Splice(X)`-Knoten löst die äußere Hülle von $X$ auf und fügt dessen Elemente direkt in den Ziel-Container ein. + +### 3.1 Striktes Verhalten (Entscheidung) + +In dieser Implementierung gilt ein **striktes** Splicing. Das bedeutet, dass $X$ zur Kompilierzeit als Container erkannt werden muss. + +- **In Sequenzen (Tuple / Block):** + - $X$ muss ein `Tuple` oder ein `Block` sein. + - Die Elemente von $X$ werden flach in die Ziel-Sequenz eingefügt. +- **In Records (Map):** + - $X$ muss eine `Map` sein. + - Die Key-Value-Paare von $X$ werden mit der Ziel-Map verschmolzen (Merging). +- **Fehlverhalten:** Falls $X$ kein unterstützter Container ist (z.B. eine Konstante oder ein If-Ausdruck), bricht der Compiler mit einem Fehler ab. Dies verhindert logische Inkonsistenzen im Makro-Design. + +## 4. Transparenz & Debugging (Der Expansion-Knoten) + +Der `Expansion`-Knoten ist das Herzstück für ein sauberes User-Erlebnis: + +1. **Origin-Tracking:** Er behält die `Identity` (Span/NodeId) des ursprünglichen Aufrufs. +2. **Transparenz:** Die UI kann im Fehlerfall auf den High-Level-Block verweisen, während der Compiler den hocheffizienten, expandierten Code sieht. +3. **Visualisierung:** Er ermöglicht ein "Aufklappen" von Makro-Blöcken in der GUI, um die innere Funktionsweise des expandierten Codes zu inspizieren. diff --git a/examples/closure.myc b/examples/closure.myc index 0efd59d..bf02503 100644 --- a/examples/closure.myc +++ b/examples/closure.myc @@ -1,5 +1,5 @@ ;; Closure Scope Test -;; Benchmark: 9.4us +;; Benchmark: 600ns ;; Output: 15 (do (def make-adder (fn [x] diff --git a/examples/data.myc b/examples/data.myc index 7f3e48e..7171ca0 100644 --- a/examples/data.myc +++ b/examples/data.myc @@ -1,5 +1,5 @@ ;; Complex Data Structure Test -;; Benchmark: 64.4us +;; Benchmark: 49.8us ;; Output: {:input 10, :name "Fibonacci", :output 55} (do (def fib (fn [n] diff --git a/examples/extreme_capture.myc b/examples/extreme_capture.myc index fb77389..9b80bfc 100644 --- a/examples/extreme_capture.myc +++ b/examples/extreme_capture.myc @@ -1,4 +1,4 @@ -;; Benchmark: 25.9us +;; Benchmark: 1.5us ;; Output: 36 (do ;; Excessive capture test diff --git a/examples/fib.myc b/examples/fib.myc index 8412ec6..0e07f46 100644 --- a/examples/fib.myc +++ b/examples/fib.myc @@ -1,5 +1,5 @@ ;; Fibonacci Recursive -;; Benchmark: 60.3us +;; Benchmark: 49.7us ;; Output: 55 (do (def fib (fn [n] diff --git a/examples/hof.myc b/examples/hof.myc index 367e052..35d16de 100644 --- a/examples/hof.myc +++ b/examples/hof.myc @@ -1,5 +1,5 @@ ;; Higher-Order Function Example -;; Benchmark: 8.9us +;; Benchmark: 600ns ;; Output: 25 (do (def apply (fn [f x] (f x))) diff --git a/examples/macro_financial.myc b/examples/macro_financial.myc new file mode 100644 index 0000000..f6472d7 --- /dev/null +++ b/examples/macro_financial.myc @@ -0,0 +1,8 @@ +;; Benchmark: 400ns +;; Financial DSL Macro example +;; Demonstrates generating complex records from simple parameters. +;; Output: {:price 100, :size 2, :total 200} + +(do + (macro trade [p s] `{:price ~p :size ~s :total (* ~p ~s)}) + (trade 100 2)) diff --git a/examples/macro_power.myc b/examples/macro_power.myc new file mode 100644 index 0000000..c1c85b3 --- /dev/null +++ b/examples/macro_power.myc @@ -0,0 +1,8 @@ +;; Benchmark: 300ns +;; Nested Macro and Arithmetic example +;; Output: 81 + +(do + (macro square [x] `(* ~x ~x)) + (macro power4 [x] `(square (square ~x))) + (power4 3)) diff --git a/examples/macro_splice.myc b/examples/macro_splice.myc new file mode 100644 index 0000000..a6f4a28 --- /dev/null +++ b/examples/macro_splice.myc @@ -0,0 +1,8 @@ +;; Benchmark: 200ns +;; Macro Splicing example +;; Demonstrates unrolling a list into another list. +;; Output: [0 1 2 3 4] + +(do + (macro wrap [items] `[0 ~@items 4]) + (wrap [1 2 3])) diff --git a/examples/macro_unless.myc b/examples/macro_unless.myc new file mode 100644 index 0000000..f591a88 --- /dev/null +++ b/examples/macro_unless.myc @@ -0,0 +1,8 @@ +;; Benchmark: 100ns +;; Classic "unless" macro example +;; Demonstrates simple template substitution. +;; Output: 42 + +(do + (macro unless [c b] `(if ~c void ~b)) + (unless false 42)) diff --git a/src/ast/compiler/binder.rs b/src/ast/compiler/binder.rs index e8ea046..ee16f29 100644 --- a/src/ast/compiler/binder.rs +++ b/src/ast/compiler/binder.rs @@ -300,6 +300,19 @@ impl Binder { Ok(self.make_node(node.identity.clone(), BoundKind::Map { entries: bound_entries }, ty)) }, + + UntypedKind::Expansion { call, expanded } => { + let bound_expanded = self.bind(expanded)?; + let ty = bound_expanded.ty.clone(); + Ok(self.make_node(node.identity.clone(), BoundKind::Expansion { + original_call: Rc::from(call.as_ref().clone()), + bound_expanded: Box::new(bound_expanded), + }, ty)) + }, + + UntypedKind::Template(_) | UntypedKind::Placeholder(_) | UntypedKind::Splice(_) | UntypedKind::MacroDecl { .. } => { + Err(format!("Macro construct {:?} found in Binder. Macros must be expanded before binding.", node.kind)) + } UntypedKind::Extension(_) => { Err("Custom extensions not supported in Binder yet".to_string()) diff --git a/src/ast/compiler/bound_nodes.rs b/src/ast/compiler/bound_nodes.rs index fbcd69c..fd73bda 100644 --- a/src/ast/compiler/bound_nodes.rs +++ b/src/ast/compiler/bound_nodes.rs @@ -70,6 +70,13 @@ pub enum BoundKind { Map { entries: Vec<(Node, Node)>, }, + /// An expanded macro call, preserving the original call for debugging and UI. + Expansion { + /// The original call from the untyped AST. + original_call: Rc>, + /// The result of binding the expanded AST. + bound_expanded: Box>, + }, } impl BoundKind { @@ -88,6 +95,7 @@ impl BoundKind { BoundKind::Block { .. } => "BLOCK".to_string(), BoundKind::Tuple { elements } => format!("TUPLE({})", elements.len()), BoundKind::Map { entries } => format!("MAP({})", entries.len()), + BoundKind::Expansion { .. } => "EXPANSION".to_string(), } } } diff --git a/src/ast/compiler/dumper.rs b/src/ast/compiler/dumper.rs index 93d7dce..9aa7768 100644 --- a/src/ast/compiler/dumper.rs +++ b/src/ast/compiler/dumper.rs @@ -149,6 +149,13 @@ impl Dumper { } self.indent -= 1; } + + BoundKind::Expansion { original_call, bound_expanded } => { + self.log(&format!("Expansion (Original: {:?})", original_call.kind), node); + self.indent += 1; + self.visit(bound_expanded); + self.indent -= 1; + } } } } diff --git a/src/ast/compiler/macros.rs b/src/ast/compiler/macros.rs new file mode 100644 index 0000000..58d228e --- /dev/null +++ b/src/ast/compiler/macros.rs @@ -0,0 +1,453 @@ +use std::collections::HashMap; +use std::rc::Rc; +use crate::ast::nodes::{Node, UntypedKind}; +use crate::ast::types::{Value, Identity}; + +/// Trait for evaluating expressions during macro expansion. +/// Similar to TMacroEvaluatorProc in the Delphi implementation. +pub trait MacroEvaluator { + fn evaluate(&self, node: &Node, bindings: &HashMap, Node>) -> Result; +} + +/// A registry for macro declarations. +pub struct MacroRegistry { + scopes: Vec, (Vec>, Node)>>, +} + +impl MacroRegistry { + pub fn new() -> Self { + Self { + scopes: vec![HashMap::new()], + } + } + + pub fn push(&mut self) { + self.scopes.push(HashMap::new()); + } + + pub fn pop(&mut self) { + if self.scopes.len() > 1 { + self.scopes.pop(); + } + } + + pub fn define(&mut self, name: Rc, params: Vec>, body: Node) { + if let Some(current) = self.scopes.last_mut() { + current.insert(name, (params, body)); + } + } + + pub fn lookup(&self, name: &str) -> Option<(Vec>, Node)> { + for scope in self.scopes.iter().rev() { + if let Some(m) = scope.get(name) { + return Some(m.clone()); + } + } + None + } +} + +pub struct MacroExpander { + registry: MacroRegistry, + evaluator: E, +} + +impl MacroExpander { + pub fn new(registry: MacroRegistry, evaluator: E) -> Self { + Self { registry, evaluator } + } + + pub fn expand(&mut self, node: Node) -> Result, String> { + self.expand_recursive(node) + } + + fn expand_recursive(&mut self, node: Node) -> Result, String> { + match node.kind { + UntypedKind::MacroDecl { name, params, body } => { + self.registry.define(name, params, *body); + Ok(Node { + identity: node.identity, + kind: UntypedKind::Nop, + ty: (), + }) + } + + UntypedKind::Call { callee, args } => { + if let UntypedKind::Identifier(ref name) = callee.kind { + if let Some((params, body)) = self.registry.lookup(name) { + let mut expanded_args = Vec::new(); + for arg in args { + expanded_args.push(self.expand_recursive(arg)?); + } + let expanded = self.expand_call(node.identity.clone(), name, params, expanded_args, body)?; + return self.expand_recursive(expanded); + } + } + + let expanded_callee = self.expand_recursive(*callee)?; + let mut expanded_args = Vec::new(); + for arg in args { + expanded_args.push(self.expand_recursive(arg)?); + } + + Ok(Node { + identity: node.identity, + kind: UntypedKind::Call { + callee: Box::new(expanded_callee), + args: expanded_args, + }, + ty: (), + }) + } + + UntypedKind::Block { exprs } => { + self.registry.push(); + let mut expanded_exprs = Vec::new(); + for expr in exprs { + expanded_exprs.push(self.expand_recursive(expr)?); + } + self.registry.pop(); + + Ok(Node { + identity: node.identity, + kind: UntypedKind::Block { exprs: expanded_exprs }, + ty: (), + }) + } + + UntypedKind::Lambda { params, body } => { + self.registry.push(); + let expanded_body = self.expand_recursive(Node::clone(&body))?; + self.registry.pop(); + + Ok(Node { + identity: node.identity, + kind: UntypedKind::Lambda { params, body: Rc::new(expanded_body) }, + ty: (), + }) + } + + UntypedKind::If { cond, then_br, else_br } => { + let cond = self.expand_recursive(*cond)?; + let then_br = self.expand_recursive(*then_br)?; + let else_br = if let Some(e) = else_br { + Some(Box::new(self.expand_recursive(*e)?)) + } else { + None + }; + Ok(Node { + identity: node.identity, + kind: UntypedKind::If { cond: Box::new(cond), then_br: Box::new(then_br), else_br }, + ty: (), + }) + } + + UntypedKind::Def { name, value } => { + let value = self.expand_recursive(*value)?; + Ok(Node { + identity: node.identity, + kind: UntypedKind::Def { name, value: Box::new(value) }, + ty: (), + }) + } + + UntypedKind::Assign { target, value } => { + let target = self.expand_recursive(*target)?; + let value = self.expand_recursive(*value)?; + Ok(Node { + identity: node.identity, + kind: UntypedKind::Assign { target: Box::new(target), value: Box::new(value) }, + ty: (), + }) + } + + UntypedKind::Tuple { elements } => { + let mut expanded_elements = Vec::new(); + for e in elements { + expanded_elements.push(self.expand_recursive(e)?); + } + Ok(Node { + identity: node.identity, + kind: UntypedKind::Tuple { elements: expanded_elements }, + ty: (), + }) + } + + UntypedKind::Map { entries } => { + let mut expanded_entries = Vec::new(); + for (k, v) in entries { + expanded_entries.push((self.expand_recursive(k)?, self.expand_recursive(v)?)); + } + Ok(Node { + identity: node.identity, + kind: UntypedKind::Map { entries: expanded_entries }, + ty: (), + }) + } + + UntypedKind::Expansion { call, expanded } => { + let expanded = self.expand_recursive(*expanded)?; + Ok(Node { + identity: node.identity, + kind: UntypedKind::Expansion { call, expanded: Box::new(expanded) }, + ty: (), + }) + } + + _ => Ok(node), + } + } + + fn expand_call(&self, identity: Identity, name: &str, params: Vec>, args: Vec>, body: Node) -> Result, String> { + if params.len() != args.len() { + return Err(format!("Macro {} expects {} arguments, but got {}", name, params.len(), args.len())); + } + + let mut bindings = HashMap::new(); + for (p, a) in params.into_iter().zip(args.into_iter()) { + bindings.insert(p, a); + } + + let expanded_body = if let UntypedKind::Template(inner) = body.kind { + self.expand_template(*inner, &bindings)? + } else { + body + }; + + let original_call = Node { + identity: identity.clone(), + kind: UntypedKind::Call { + callee: Box::new(Node { + identity: identity.clone(), + kind: UntypedKind::Identifier(Rc::from(name)), + ty: (), + }), + args: bindings.iter().map(|(_, v)| v.clone()).collect(), + }, + ty: (), + }; + + Ok(Node { + identity, + kind: UntypedKind::Expansion { + call: Box::new(original_call), + expanded: Box::new(expanded_body), + }, + ty: (), + }) + } + + fn expand_template(&self, node: Node, bindings: &HashMap, Node>) -> Result, String> { + match node.kind { + UntypedKind::Placeholder(inner) => { + let val = self.evaluator.evaluate(&inner, bindings)?; + Ok(self.value_to_node(val, node.identity)) + } + + UntypedKind::Splice(_) => { + Err("Splice node found outside of a container context".to_string()) + } + + UntypedKind::Tuple { elements } => { + let mut new_elements = Vec::new(); + for e in elements { + if let UntypedKind::Splice(ref inner) = e.kind { + let val = self.evaluator.evaluate(inner, bindings)?; + self.handle_splice_value(val, node.identity.clone(), &mut new_elements)?; + } else { + new_elements.push(self.expand_template(e, bindings)?); + } + } + Ok(Node { + identity: node.identity, + kind: UntypedKind::Tuple { elements: new_elements }, + ty: (), + }) + } + + UntypedKind::Block { exprs } => { + let mut new_exprs = Vec::new(); + for e in exprs { + if let UntypedKind::Splice(ref inner) = e.kind { + let val = self.evaluator.evaluate(inner, bindings)?; + self.handle_splice_value(val, node.identity.clone(), &mut new_exprs)?; + } else { + new_exprs.push(self.expand_template(e, bindings)?); + } + } + Ok(Node { + identity: node.identity, + kind: UntypedKind::Block { exprs: new_exprs }, + ty: (), + }) + } + + UntypedKind::Map { entries } => { + let mut new_entries = Vec::new(); + for (k, v) in entries { + new_entries.push((self.expand_template(k, bindings)?, self.expand_template(v, bindings)?)); + } + Ok(Node { + identity: node.identity, + kind: UntypedKind::Map { entries: new_entries }, + ty: (), + }) + } + + UntypedKind::Call { callee, args } => { + let expanded_callee = self.expand_template(*callee, bindings)?; + let mut expanded_args = Vec::new(); + for arg in args { + if let UntypedKind::Splice(ref inner) = arg.kind { + let val = self.evaluator.evaluate(inner, bindings)?; + self.handle_splice_value(val, node.identity.clone(), &mut expanded_args)?; + } else { + expanded_args.push(self.expand_template(arg, bindings)?); + } + } + Ok(Node { + identity: node.identity, + kind: UntypedKind::Call { callee: Box::new(expanded_callee), args: expanded_args }, + ty: (), + }) + } + + UntypedKind::If { cond, then_br, else_br } => { + let cond = self.expand_template(*cond, bindings)?; + let then_br = self.expand_template(*then_br, bindings)?; + let else_br = if let Some(e) = else_br { + Some(Box::new(self.expand_template(*e, bindings)?)) + } else { + None + }; + Ok(Node { + identity: node.identity, + kind: UntypedKind::If { cond: Box::new(cond), then_br: Box::new(then_br), else_br }, + ty: (), + }) + } + + _ => Ok(node), + } + } + + fn handle_splice_value(&self, val: Value, _identity: Identity, target: &mut Vec>) -> Result<(), String> { + match val { + Value::Object(obj) => { + if let Some(node) = obj.as_any().downcast_ref::>() { + match &node.kind { + UntypedKind::Tuple { elements } => { + for e in elements { target.push(e.clone()); } + return Ok(()); + } + UntypedKind::Block { exprs } => { + for e in exprs { target.push(e.clone()); } + return Ok(()); + } + _ => {} + } + } + Err(format!("Strict Splicing: Expected Tuple or Block node, but found {}", obj.type_name())) + } + _ => Err(format!("Strict Splicing: Expected sequence (Tuple/Block Node), but found Value: {}", val)), + } + } + + fn value_to_node(&self, val: Value, identity: Identity) -> Node { + match val { + Value::Object(obj) => { + if let Some(node) = obj.as_any().downcast_ref::>() { + return node.clone(); + } + Node { identity, kind: UntypedKind::Constant(Value::Object(obj)), ty: () } + } + _ => Node { identity, kind: UntypedKind::Constant(val), ty: () } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::parser::Parser; + use crate::ast::types::Value; + + struct SimpleEvaluator; + impl MacroEvaluator for SimpleEvaluator { + fn evaluate(&self, node: &Node, bindings: &HashMap, Node>) -> Result { + if let UntypedKind::Identifier(ref name) = node.kind { + if let Some(arg) = bindings.get(name) { + return Ok(Value::Object(Rc::new(arg.clone()))); + } + } + // For real expressions, we would need Binder + VM here. + Err(format!("SimpleEvaluator cannot evaluate complex expression: {:?}", node.kind)) + } + } + + #[test] + fn test_macro_expansion_unless() { + let source = " + (do + (macro unless [c b] `(if ~c void ~b)) + (unless true 42)) + "; + let mut parser = Parser::new(source).unwrap(); + let untyped = parser.parse_expression().unwrap(); + + let mut expander = MacroExpander::new(MacroRegistry::new(), SimpleEvaluator); + let expanded = expander.expand(untyped).unwrap(); + + if let UntypedKind::Block { exprs } = &expanded.kind { + assert_eq!(exprs.len(), 2); + assert!(matches!(exprs[0].kind, UntypedKind::Nop)); + + if let UntypedKind::Expansion { call: _, expanded: result } = &exprs[1].kind { + if let UntypedKind::If { cond, then_br, else_br } = &result.kind { + if let UntypedKind::Constant(Value::Bool(b)) = &cond.kind { + assert_eq!(*b, true); + } else { panic!("Expected true, got {:?}", cond.kind); } + assert!(matches!(then_br.kind, UntypedKind::Constant(Value::Void))); + if let Some(eb) = else_br { + if let UntypedKind::Constant(Value::Int(n)) = &eb.kind { + assert_eq!(*n, 42); + } else { panic!("Expected 42, got {:?}", eb.kind); } + } else { panic!("Else branch missing"); } + } else { + panic!("Expansion result should be an IF, got {:?}", result.kind); + } + } else { + panic!("Second expression should be an Expansion, got {:?}", exprs[1].kind); + } + } else { + panic!("Root should be a Block"); + } + } + + #[test] + fn test_macro_expansion_splice() { + let source = " + (do + (macro wrap [items] `[0 ~@items 4]) + (def t (wrap [1 2 3]))) + "; + let mut parser = Parser::new(source).unwrap(); + let untyped = parser.parse_expression().unwrap(); + + let mut expander = MacroExpander::new(MacroRegistry::new(), SimpleEvaluator); + let expanded = expander.expand(untyped).unwrap(); + + if let UntypedKind::Block { exprs } = &expanded.kind { + if let UntypedKind::Def { value, .. } = &exprs[1].kind { + if let UntypedKind::Expansion { expanded: result, .. } = &value.kind { + if let UntypedKind::Tuple { elements } = &result.kind { + assert_eq!(elements.len(), 5); + if let UntypedKind::Constant(Value::Int(n)) = &elements[0].kind { assert_eq!(*n, 0); } + if let UntypedKind::Constant(Value::Int(n)) = &elements[1].kind { assert_eq!(*n, 1); } + if let UntypedKind::Constant(Value::Int(n)) = &elements[4].kind { assert_eq!(*n, 4); } + } else { panic!("Expected Tuple result, got {:?}", result.kind); } + } else { panic!("Expected Expansion for t, got {:?}", value.kind); } + } + } + } +} diff --git a/src/ast/compiler/mod.rs b/src/ast/compiler/mod.rs index 9dfce23..f74ee58 100644 --- a/src/ast/compiler/mod.rs +++ b/src/ast/compiler/mod.rs @@ -3,9 +3,11 @@ pub mod bound_nodes; pub mod tco; pub mod upvalues; pub mod dumper; +pub mod macros; pub use binder::*; pub use bound_nodes::*; pub use tco::*; pub use upvalues::*; pub use dumper::*; +pub use macros::*; diff --git a/src/ast/compiler/upvalues.rs b/src/ast/compiler/upvalues.rs index 380a7c1..0f43f81 100644 --- a/src/ast/compiler/upvalues.rs +++ b/src/ast/compiler/upvalues.rs @@ -93,6 +93,12 @@ impl UpvalueAnalyzer { Self::visit(v, scopes, capture_map, current_lambda.clone()); } } + UntypedKind::Expansion { call: _, expanded } => { + Self::visit(expanded, scopes, capture_map, current_lambda); + } + UntypedKind::MacroDecl { body, .. } | UntypedKind::Template(body) | UntypedKind::Placeholder(body) | UntypedKind::Splice(body) => { + Self::visit(body, scopes, capture_map, current_lambda); + } UntypedKind::Nop | UntypedKind::Constant(_) | UntypedKind::Extension(_) => {} } } diff --git a/src/ast/environment.rs b/src/ast/environment.rs index 9a94b45..c81d549 100644 --- a/src/ast/environment.rs +++ b/src/ast/environment.rs @@ -1,14 +1,16 @@ use std::rc::Rc; use std::cell::RefCell; use std::collections::HashMap; -use crate::ast::types::{Value, StaticType}; +use crate::ast::types::{Value, StaticType, Object}; +use crate::ast::nodes::{Node, UntypedKind}; use crate::ast::parser::Parser; use crate::ast::compiler::binder::Binder; +use crate::ast::compiler::BoundKind; use crate::ast::vm::{VM, TracingObserver}; use crate::ast::compiler::tco::TCO; - use crate::ast::compiler::dumper::Dumper; +use crate::ast::compiler::macros::{MacroExpander, MacroRegistry, MacroEvaluator}; pub struct Environment { pub global_names: Rc>>, @@ -16,6 +18,30 @@ pub struct Environment { pub debug_mode: bool, } +/// Evaluator used during macro expansion to allow compile-time logic. +struct RuntimeMacroEvaluator { + global_names: Rc>>, + global_values: Rc>>, +} + +impl MacroEvaluator for RuntimeMacroEvaluator { + fn evaluate(&self, node: &Node, bindings: &HashMap, Node>) -> Result { + // 1. Check if it's a simple parameter substitution + if let UntypedKind::Identifier(name) = &node.kind { + if let Some(arg_node) = bindings.get(name) { + return Ok(Value::Object(Rc::new(arg_node.clone()) as Rc)); + } + } + + // 2. Full evaluation for complex compile-time expressions + // Note: This evaluator currently doesn't see macro parameters in complex expressions + // unless they are explicitly substituted. For now, we support global-only logic. + let bound_ast = Binder::bind_root(self.global_names.clone(), node)?; + let mut vm = VM::new(self.global_values.clone()); + vm.run(&bound_ast) + } +} + impl Default for Environment { fn default() -> Self { Self::new() @@ -37,6 +63,14 @@ impl Environment { self.debug_mode = enabled; } + fn get_expander(&self) -> MacroExpander { + let evaluator = RuntimeMacroEvaluator { + global_names: self.global_names.clone(), + global_values: self.global_values.clone(), + }; + MacroExpander::new(MacroRegistry::new(), evaluator) + } + pub fn register_native(&self, name: &str, ty: StaticType, func: impl Fn(Vec) -> Value + 'static) { let mut names = self.global_names.borrow_mut(); let mut values = self.global_values.borrow_mut(); @@ -114,13 +148,38 @@ impl Environment { } pub fn dump_ast(&self, source: &str) -> Result { + let compiled = self.compile(source)?; + let linked = self.link(compiled); + Ok(Dumper::dump(&linked)) + } + + /// Frontend: Parse -> Expand Macros -> Bind & Type Check + pub fn compile(&self, source: &str) -> Result, String> { + // 1. Parse let mut parser = Parser::new(source)?; let untyped_ast = parser.parse_expression()?; - - let bound_ast = Binder::bind_root(self.global_names.clone(), &untyped_ast)?; - let optimized_ast = TCO::optimize(bound_ast); - - Ok(Dumper::dump(&optimized_ast)) + + // 2. Check for trailing tokens + if !parser.at_eof() { + return Err("Unexpected trailing expressions in script. Use (do ...) for sequences.".to_string()); + } + + // 3. Expand Macros + let expanded_ast = self.get_expander().expand(untyped_ast)?; + + // 4. Bind & Type Check + Binder::bind_root(self.global_names.clone(), &expanded_ast) + } + + /// Backend: Optimization (TCO, etc.) + pub fn link(&self, node: Node) -> Node { + TCO::optimize(node) + } + + /// Runtime: Execute the linked AST in the VM + pub fn run(&self, node: &Node) -> Result { + let mut vm = VM::new(self.global_values.clone()); + vm.run(node) } pub fn run_script(&self, source: &str) -> Result { @@ -131,42 +190,20 @@ impl Environment { } res } else { - // 1. Parse - let mut parser = Parser::new(source)?; - let untyped_ast = parser.parse_expression()?; - - // 2. Check for trailing tokens - if !parser.at_eof() { - return Err("Unexpected trailing expressions in script. Use (do ...) for sequences.".to_string()); - } - - // 3. Bind & Type Check - let bound_ast = Binder::bind_root(self.global_names.clone(), &untyped_ast)?; - - // 4. Optimize - let bound_ast = TCO::optimize(bound_ast); - - // 5. Execute - let mut vm = VM::new(self.global_values.clone()); - vm.run(&bound_ast) + let compiled = self.compile(source)?; + let linked = self.link(compiled); + self.run(&linked) } } pub fn run_debug(&self, source: &str) -> Result<(Result, Vec), String> { - // 1. Parse - let mut parser = Parser::new(source)?; - let untyped_ast = parser.parse_expression()?; + let compiled = self.compile(source)?; + let linked = self.link(compiled); - // 2. Bind - let bound_ast = Binder::bind_root(self.global_names.clone(), &untyped_ast)?; - - // 3. Optimize (TCO) - let bound_ast = TCO::optimize(bound_ast); - - // 4. Execute with TracingObserver + // Execute with TracingObserver let mut vm = VM::new(self.global_values.clone()); let mut observer = TracingObserver::new(); - let result = vm.run_with_observer(&mut observer, &bound_ast); + let result = vm.run_with_observer(&mut observer, &linked); Ok((result, observer.logs)) } diff --git a/src/ast/lexer.rs b/src/ast/lexer.rs index 18edaf9..49f5000 100644 --- a/src/ast/lexer.rs +++ b/src/ast/lexer.rs @@ -102,11 +102,12 @@ impl<'a> Lexer<'a> { fn skip_whitespace(&mut self) { while let Some(&c) = self.peek() { - if c.is_whitespace() { + if c.is_whitespace() || is_invisible(c) { if c == '\n' { self.line += 1; self.col = 1; - } else { + } else if !is_invisible(c) { + // Only increment column for characters that actually take space self.col += 1; } self.input.next(); @@ -175,3 +176,17 @@ fn is_ident_start(c: char) -> bool { fn is_ident_char(c: char) -> bool { is_ident_start(c) || c.is_ascii_digit() } + +/// Returns true if the character is an invisible control character +/// that should be ignored by the lexer (like BOM or Zero Width Space). +fn is_invisible(c: char) -> bool { + match c { + '\u{FEFF}' | // BOM + '\u{200B}' | // Zero Width Space + '\u{200C}' | // Zero Width Non-Joiner + '\u{200D}' | // Zero Width Joiner + '\u{2060}' // Word Joiner + => true, + _ => false, + } +} diff --git a/src/ast/nodes.rs b/src/ast/nodes.rs index 82e0628..90fa71a 100644 --- a/src/ast/nodes.rs +++ b/src/ast/nodes.rs @@ -1,6 +1,7 @@ use std::rc::Rc; use std::fmt::Debug; -use crate::ast::types::{Identity, Value}; +use std::any::Any; +use crate::ast::types::{Identity, Value, Object}; /// A generic AST Node wrapper to preserve identity and metadata #[derive(Debug, Clone)] @@ -10,12 +11,28 @@ pub struct Node { pub ty: T, } +impl Object for Node { + fn type_name(&self) -> &'static str { + "ast-node" + } + fn as_any(&self) -> &dyn Any { + self + } +} + /// The base for custom node types (extensions) pub trait CustomNode: Debug { fn display_name(&self) -> &'static str; + fn clone_box(&self) -> Box; } -#[derive(Debug)] +impl Clone for Box { + fn clone(&self) -> Self { + self.clone_box() + } +} + +#[derive(Debug, Clone)] pub enum UntypedKind { Nop, Constant(Value), @@ -50,5 +67,24 @@ pub enum UntypedKind { Map { entries: Vec<(Node, Node)>, }, + /// A macro declaration that can be expanded at compile time. + MacroDecl { + name: Rc, + params: Vec>, + body: Box>, + }, + /// A template for AST nodes, allowing for substitutions. (Quasiquote) + Template(Box>), + /// A placeholder inside a template to be replaced by a node or value. (Unquote) + Placeholder(Box>), + /// A placeholder that flattens a sequence or merges a map into its surroundings. (Splice) + Splice(Box>), + /// Represents an expanded macro call, preserving the original call for debugging. + Expansion { + /// The original call from the source AST. + call: Box>, + /// The resulting AST after macro expansion. + expanded: Box>, + }, Extension(Box), } diff --git a/src/ast/parser.rs b/src/ast/parser.rs index 4a6dfe3..12d935c 100644 --- a/src/ast/parser.rs +++ b/src/ast/parser.rs @@ -25,12 +25,14 @@ impl<'a> Parser<'a> { } pub fn parse_expression(&mut self) -> Result, String> { + let token_loc = self.current_token.location; + let identity = Rc::new(NodeIdentity { location: token_loc }); + match self.peek() { TokenKind::LeftParen => self.parse_list(), TokenKind::LeftBracket => self.parse_vector_literal(), TokenKind::LeftBrace => self.parse_map_literal(), TokenKind::Quote => { - let identity = Rc::new(NodeIdentity { location: self.current_token.location }); self.advance()?; // consume ' let expr = self.parse_expression()?; Ok(Node { @@ -42,6 +44,34 @@ impl<'a> Parser<'a> { ty: (), }) } + TokenKind::Backtick => { + self.advance()?; // consume ` + let expr = self.parse_expression()?; + Ok(Node { + identity, + kind: UntypedKind::Template(Box::new(expr)), + ty: (), + }) + } + TokenKind::Tilde => { + self.advance()?; // consume ~ + if *self.peek() == TokenKind::At { + self.advance()?; // consume @ + let expr = self.parse_expression()?; + Ok(Node { + identity, + kind: UntypedKind::Splice(Box::new(expr)), + ty: (), + }) + } else { + let expr = self.parse_expression()?; + Ok(Node { + identity, + kind: UntypedKind::Placeholder(Box::new(expr)), + ty: (), + }) + } + } _ => self.parse_atom(), } } @@ -91,6 +121,7 @@ impl<'a> Parser<'a> { "def" => self.parse_def(identity), "assign" => self.parse_assign(identity), "do" => self.parse_do(identity), + "macro" => self.parse_macro_decl(identity), _ => self.parse_call(head, identity), } } else { @@ -171,6 +202,21 @@ impl<'a> Parser<'a> { }) } + fn parse_macro_decl(&mut self, identity: Identity) -> Result, String> { + let name_node = self.parse_expression()?; + let name = match name_node.kind { + UntypedKind::Identifier(s) => s, + _ => return Err("Expected identifier for macro name".to_string()), + }; + let params = self.parse_param_vector()?; + let body = self.parse_expression()?; + Ok(Node { + identity, + kind: UntypedKind::MacroDecl { name, params, body: Box::new(body) }, + ty: (), + }) + } + fn parse_param_vector(&mut self) -> Result>, String> { if *self.peek() != TokenKind::LeftBracket { return Err(format!("Expected parameter vector [...] for fn, found {:?}", self.peek())); diff --git a/src/ast/tester.rs b/src/ast/tester.rs index 813724b..6b5dcdd 100644 --- a/src/ast/tester.rs +++ b/src/ast/tester.rs @@ -66,11 +66,17 @@ pub fn run_benchmarks(update: bool) -> Vec { let baseline_match = baseline_re.captures(&content); - // Measure + // Prepare: Compile and Link once outside the measurement loop + let linked_node = match env.compile(&content).map(|c| env.link(c)) { + Ok(node) => node, + Err(_) => continue, // Skip scripts that don't compile + }; + + // Measure: Only the VM execution let mut runs = Vec::new(); for _ in 0..100 { let start = Instant::now(); - let _ = env.run_script(&content); + let _ = env.run(&linked_node); runs.push(start.elapsed()); } runs.sort(); diff --git a/src/ast/vm.rs b/src/ast/vm.rs index fc9ddd8..8e97000 100644 --- a/src/ast/vm.rs +++ b/src/ast/vm.rs @@ -258,6 +258,10 @@ macro_rules! dispatch_eval { } } Ok(Value::Record(Rc::new(map))) + }, + + BoundKind::Expansion { original_call: _, bound_expanded } => { + $self.$eval_method($($observer,)? bound_expanded) } } }; diff --git a/src/main.rs b/src/main.rs index 30d368b..b4643ce 100644 --- a/src/main.rs +++ b/src/main.rs @@ -84,30 +84,45 @@ impl Default for CompilerApp { impl CompilerApp { fn compile(&mut self) { - let start = std::time::Instant::now(); + let start_total = std::time::Instant::now(); self.trace_logs.clear(); - let result = if self.debug_enabled { - match self.env.run_debug(&self.source_code) { - Ok((res, logs)) => { - self.trace_logs = logs; - self.active_tab = AppTab::Trace; - res - } - Err(e) => Err(e), - } - } else { - self.env.run_script(&self.source_code) - }; + let res = (|| -> Result<(myc::ast::types::Value, std::time::Duration, std::time::Duration, std::time::Duration), String> { + let start_compile = std::time::Instant::now(); + let compiled = self.env.compile(&self.source_code)?; + let duration_compile = start_compile.elapsed(); - match result { - Ok(val) => { - let duration = start.elapsed(); + let start_link = std::time::Instant::now(); + let linked = self.env.link(compiled); + let duration_link = start_link.elapsed(); + + let start_run = std::time::Instant::now(); + let result = if self.debug_enabled { + let mut vm = myc::ast::vm::VM::new(self.env.global_values.clone()); + let mut observer = myc::ast::vm::TracingObserver::new(); + let res = vm.run_with_observer(&mut observer, &linked); + self.trace_logs = observer.logs; + self.active_tab = AppTab::Trace; + res? + } else { + self.env.run(&linked)? + }; + let duration_run = start_run.elapsed(); + + Ok((result, duration_compile, duration_link, duration_run)) + })(); + + match res { + Ok((val, d_compile, d_link, d_run)) => { + let d_total = start_total.elapsed(); let now = chrono::Local::now().format("%H:%M:%S").to_string(); self.output_log = format!( - "Execution Successful.\nResult: {}\n\nDuration: {:?}\nFinished at: {}", + "Execution Successful.\nResult: {}\n\nPhases:\n - Compile: {:?}\n - Link: {:?}\n - Run: {:?}\n\nTotal Duration: {:?}\nFinished at: {}", val, - duration, + d_compile, + d_link, + d_run, + d_total, now, ); if !self.debug_enabled { @@ -466,6 +481,9 @@ fn highlight_myc(ctx: &egui::Context, code: &str) -> egui::text::LayoutJob { bracket_depth = bracket_depth.saturating_sub(1); color = color_bracket[bracket_depth % 3]; } + '`' | '~' | '@' => { + color = color_kw; + } '0'..='9' => { while let Some(&next) = chars.peek() { if next.is_ascii_digit() || next == '.' { @@ -480,7 +498,7 @@ fn highlight_myc(ctx: &egui::Context, code: &str) -> egui::text::LayoutJob { text.push(chars.next().unwrap()); } else { break; } } - if ["fn", "def", "do", "if", "assign", "recur", "cond"].contains(&text.as_str()) { + if ["fn", "def", "do", "if", "assign", "recur", "cond", "macro"].contains(&text.as_str()) { color = color_kw; } }