Add AST dumper and improve upvalue analysis
The `UpvalueAnalyzer` now correctly identifies which lambdas capture which variable declarations, rather than just marking declarations that need boxing. This information is stored in a `capture_map`. The `Binder` has been updated to use this new `capture_map` instead of a `HashSet` of boxed declarations. The `DefLocal` node in `bound_nodes.rs` now stores a `captured_by` field, which is a list of lambda identities that capture the local variable. A new `dumper` module has been added to provide a human-readable representation of the bound AST, including information about captured variables. The `VM` has been updated to use the `captured_by` field to determine if a local variable needs to be stored in a `Cell`, rather than relying on a boolean `is_boxed` flag. An example script `extreme_capture.myc` has been added to test deep nesting and variable capture. A new "Dump AST" button has been added to the UI, which uses the new `Dumper` to display the bound AST.
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
;; Output: 36
|
||||
(do
|
||||
;; Excessive capture test
|
||||
;; This script creates deep nesting where the inner-most lambda
|
||||
;; grabs variables from every single outer scope.
|
||||
|
||||
(def excessive (fn [v1]
|
||||
(do
|
||||
(def v2 2)
|
||||
(fn [v3]
|
||||
(do
|
||||
(def v4 4)
|
||||
(fn [v5]
|
||||
(do
|
||||
(def v6 6)
|
||||
(fn [v7]
|
||||
(do
|
||||
(def v8 8)
|
||||
;; v1, v3, v5, v7 are parameters (captured as upvalues)
|
||||
;; v2, v4, v6, v8 are locals (captured from outer scopes)
|
||||
(+ v1 v2 v3 v4 v5 v6 v7 v8))))))))))
|
||||
|
||||
;; Execution: (1 + 2 + 3 + 4 + 5 + 6 + 7 + 8) = 36
|
||||
((((excessive 1) 3) 5) 7)
|
||||
)
|
||||
+14
-14
@@ -1,4 +1,4 @@
|
||||
use std::collections::{HashMap, BTreeMap, HashSet};
|
||||
use std::collections::{HashMap, BTreeMap};
|
||||
use std::rc::Rc;
|
||||
use std::cell::RefCell;
|
||||
use crate::ast::nodes::{Node, UntypedKind};
|
||||
@@ -64,28 +64,28 @@ pub struct Binder {
|
||||
functions: Vec<FunctionCompiler>,
|
||||
// Globals mapping: Name -> (Index, Type)
|
||||
globals: Rc<RefCell<HashMap<String, (u32, StaticType)>>>,
|
||||
// Identities of definitions that need boxing (pre-calculated)
|
||||
boxed_declarations: HashSet<Identity>,
|
||||
// Map of Declaration Identity -> List of Lambda Identities that capture it
|
||||
capture_map: HashMap<Identity, Vec<Identity>>,
|
||||
}
|
||||
|
||||
impl Binder {
|
||||
pub fn new(globals: Rc<RefCell<HashMap<String, (u32, StaticType)>>>) -> Self {
|
||||
Self::with_boxed(globals, HashSet::new())
|
||||
Self::with_boxed(globals, HashMap::new())
|
||||
}
|
||||
|
||||
fn with_boxed(globals: Rc<RefCell<HashMap<String, (u32, StaticType)>>>, boxed: HashSet<Identity>) -> Self {
|
||||
fn with_boxed(globals: Rc<RefCell<HashMap<String, (u32, StaticType)>>>, captures: HashMap<Identity, Vec<Identity>>) -> Self {
|
||||
let mut binder = Self {
|
||||
functions: Vec::new(),
|
||||
globals,
|
||||
boxed_declarations: boxed,
|
||||
capture_map: captures,
|
||||
};
|
||||
binder.functions.push(FunctionCompiler::new());
|
||||
binder
|
||||
}
|
||||
|
||||
pub fn bind_root(globals: Rc<RefCell<HashMap<String, (u32, StaticType)>>>, node: &Node<UntypedKind>) -> Result<Node<BoundKind, StaticType>, String> {
|
||||
let boxed = crate::ast::compiler::upvalues::UpvalueAnalyzer::analyze(node);
|
||||
let mut binder = Self::with_boxed(globals, boxed);
|
||||
let captures = crate::ast::compiler::upvalues::UpvalueAnalyzer::analyze(node);
|
||||
let mut binder = Self::with_boxed(globals, captures);
|
||||
binder.bind(node)
|
||||
}
|
||||
|
||||
@@ -170,11 +170,11 @@ impl Binder {
|
||||
info.ty = ty.clone();
|
||||
}
|
||||
}
|
||||
let is_boxed = self.boxed_declarations.contains(&node.identity);
|
||||
let captured_by = self.capture_map.get(&node.identity).cloned().unwrap_or_default();
|
||||
Ok(self.make_node(node.identity.clone(), BoundKind::DefLocal {
|
||||
slot: slot_or_idx,
|
||||
value: Box::new(val_node) ,
|
||||
is_boxed
|
||||
captured_by
|
||||
}, ty))
|
||||
}
|
||||
},
|
||||
@@ -361,8 +361,8 @@ mod tests {
|
||||
if let BoundKind::Lambda { body, .. } = &bound.kind {
|
||||
if let BoundKind::Block { exprs } = &body.kind {
|
||||
let x_decl = &exprs[0];
|
||||
if let BoundKind::DefLocal { is_boxed, .. } = &x_decl.kind {
|
||||
assert!(is_boxed, "Variable 'x' should be marked as boxed because it is captured by lambda 'f'");
|
||||
if let BoundKind::DefLocal { captured_by, .. } = &x_decl.kind {
|
||||
assert!(!captured_by.is_empty(), "Variable 'x' should have capturers because it is used in lambda 'f'");
|
||||
} else {
|
||||
panic!("First expression in block should be DefLocal, got {:?}", x_decl.kind);
|
||||
}
|
||||
@@ -386,8 +386,8 @@ mod tests {
|
||||
if let BoundKind::Lambda { body, .. } = &bound.kind {
|
||||
if let BoundKind::Block { exprs } = &body.kind {
|
||||
let x_decl = &exprs[0];
|
||||
if let BoundKind::DefLocal { is_boxed, .. } = &x_decl.kind {
|
||||
assert!(!is_boxed, "Variable 'x' should NOT be boxed as it is not captured");
|
||||
if let BoundKind::DefLocal { captured_by, .. } = &x_decl.kind {
|
||||
assert!(captured_by.is_empty(), "Variable 'x' should NOT have any capturers");
|
||||
} else {
|
||||
panic!("First expression should be DefLocal");
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use std::rc::Rc;
|
||||
use crate::ast::types::{Value, StaticType};
|
||||
use crate::ast::types::{Value, StaticType, Identity};
|
||||
use crate::ast::nodes::Node;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
@@ -27,7 +27,7 @@ pub enum BoundKind {
|
||||
DefLocal {
|
||||
slot: u32,
|
||||
value: Box<Node<BoundKind, StaticType>>,
|
||||
is_boxed: bool,
|
||||
captured_by: Vec<Identity>,
|
||||
},
|
||||
|
||||
If {
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
use crate::ast::nodes::Node;
|
||||
use crate::ast::compiler::bound_nodes::BoundKind;
|
||||
use crate::ast::types::StaticType;
|
||||
|
||||
/// Human-readable AST dumper for the bound AST.
|
||||
pub struct Dumper {
|
||||
output: String,
|
||||
indent: usize,
|
||||
}
|
||||
|
||||
impl Dumper {
|
||||
/// Produces a formatted string representation of the given bound AST node and its children.
|
||||
pub fn dump(node: &Node<BoundKind, StaticType>) -> String {
|
||||
let mut dumper = Self {
|
||||
output: String::new(),
|
||||
indent: 0,
|
||||
};
|
||||
dumper.visit(node);
|
||||
dumper.output
|
||||
}
|
||||
|
||||
fn write_indent(&mut self) {
|
||||
for _ in 0..self.indent {
|
||||
self.output.push_str(" ");
|
||||
}
|
||||
}
|
||||
|
||||
fn log(&mut self, label: &str, node: &Node<BoundKind, StaticType>) {
|
||||
self.write_indent();
|
||||
self.output.push_str(label);
|
||||
self.output.push_str(&format!(" <Type: {:?}>\n", node.ty));
|
||||
}
|
||||
|
||||
fn visit(&mut self, node: &Node<BoundKind, StaticType>) {
|
||||
match &node.kind {
|
||||
BoundKind::Nop => self.log("Nop", node),
|
||||
BoundKind::Constant(v) => self.log(&format!("Constant: {}", v), node),
|
||||
BoundKind::Get(addr) => self.log(&format!("Get: {:?}", addr), node),
|
||||
|
||||
BoundKind::Set { addr, value } => {
|
||||
self.log(&format!("Set: {:?}", addr), node);
|
||||
self.indent += 1;
|
||||
self.visit(value);
|
||||
self.indent -= 1;
|
||||
}
|
||||
|
||||
BoundKind::DefLocal { slot, value, captured_by } => {
|
||||
let capture_info = if captured_by.is_empty() {
|
||||
String::from("not captured")
|
||||
} else {
|
||||
format!("captured by {} lambdas", captured_by.len())
|
||||
};
|
||||
self.log(&format!("DefLocal (Slot: {}, {})", slot, capture_info), node);
|
||||
|
||||
self.indent += 1;
|
||||
if !captured_by.is_empty() {
|
||||
for capturer in captured_by {
|
||||
self.write_indent();
|
||||
self.output.push_str(&format!("- Capturer: Lambda at line {}, col {}\n",
|
||||
capturer.location.line, capturer.location.col));
|
||||
}
|
||||
}
|
||||
self.visit(value);
|
||||
self.indent -= 1;
|
||||
}
|
||||
|
||||
BoundKind::DefGlobal { global_index, value } => {
|
||||
self.log(&format!("DefGlobal (Index: {})", global_index), node);
|
||||
self.indent += 1;
|
||||
self.visit(value);
|
||||
self.indent -= 1;
|
||||
}
|
||||
|
||||
BoundKind::If { cond, then_br, else_br } => {
|
||||
self.log("If", node);
|
||||
self.indent += 1;
|
||||
|
||||
self.write_indent();
|
||||
self.output.push_str("Condition:\n");
|
||||
self.visit(cond);
|
||||
|
||||
self.write_indent();
|
||||
self.output.push_str("Then:\n");
|
||||
self.visit(then_br);
|
||||
|
||||
if let Some(e) = else_br {
|
||||
self.write_indent();
|
||||
self.output.push_str("Else:\n");
|
||||
self.visit(e);
|
||||
}
|
||||
self.indent -= 1;
|
||||
}
|
||||
|
||||
BoundKind::Lambda { param_count, upvalues, body } => {
|
||||
self.log(&format!("Lambda (Params: {}, Upvalues: {})", param_count, upvalues.len()), node);
|
||||
self.indent += 1;
|
||||
if !upvalues.is_empty() {
|
||||
self.write_indent();
|
||||
self.output.push_str(&format!("Upvalues: {:?}\n", upvalues));
|
||||
}
|
||||
self.visit(body);
|
||||
self.indent -= 1;
|
||||
}
|
||||
|
||||
BoundKind::Call { callee, args } => {
|
||||
self.log("Call", node);
|
||||
self.indent += 1;
|
||||
self.visit(callee);
|
||||
for arg in args {
|
||||
self.visit(arg);
|
||||
}
|
||||
self.indent -= 1;
|
||||
}
|
||||
|
||||
BoundKind::TailCall { callee, args } => {
|
||||
self.log("TailCall (TCO)", node);
|
||||
self.indent += 1;
|
||||
self.visit(callee);
|
||||
for arg in args {
|
||||
self.visit(arg);
|
||||
}
|
||||
self.indent -= 1;
|
||||
}
|
||||
|
||||
BoundKind::Block { exprs } => {
|
||||
self.log("Block", node);
|
||||
self.indent += 1;
|
||||
for expr in exprs {
|
||||
self.visit(expr);
|
||||
}
|
||||
self.indent -= 1;
|
||||
}
|
||||
|
||||
BoundKind::Tuple { elements } => {
|
||||
self.log("Tuple", node);
|
||||
self.indent += 1;
|
||||
for el in elements {
|
||||
self.visit(el);
|
||||
}
|
||||
self.indent -= 1;
|
||||
}
|
||||
|
||||
BoundKind::Map { entries } => {
|
||||
self.log("Map", node);
|
||||
self.indent += 1;
|
||||
for (k, v) in entries {
|
||||
self.visit(k);
|
||||
self.visit(v);
|
||||
}
|
||||
self.indent -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,10 @@ pub mod binder;
|
||||
pub mod bound_nodes;
|
||||
pub mod tco;
|
||||
pub mod upvalues;
|
||||
pub mod dumper;
|
||||
|
||||
pub use binder::*;
|
||||
pub use bound_nodes::*;
|
||||
pub use tco::*;
|
||||
pub use upvalues::*;
|
||||
pub use dumper::*;
|
||||
|
||||
@@ -98,9 +98,9 @@ impl TCO {
|
||||
..node
|
||||
}
|
||||
},
|
||||
BoundKind::DefLocal { slot, value, is_boxed } => {
|
||||
BoundKind::DefLocal { slot, value, captured_by } => {
|
||||
Node {
|
||||
kind: BoundKind::DefLocal { slot, value: Box::new(Self::transform(*value, false)), is_boxed },
|
||||
kind: BoundKind::DefLocal { slot, value: Box::new(Self::transform(*value, false)), captured_by },
|
||||
..node
|
||||
}
|
||||
},
|
||||
|
||||
@@ -3,85 +3,94 @@ use std::rc::Rc;
|
||||
use crate::ast::nodes::{Node, UntypedKind};
|
||||
use crate::ast::types::Identity;
|
||||
|
||||
/// Analyzes the AST to find all variable declarations that are captured by nested lambdas.
|
||||
/// Returns a set of Identities corresponding to the 'Def' nodes that need boxing.
|
||||
/// Analyzes the AST to find which lambdas capture which variable declarations.
|
||||
/// Returns a map: Declaration Identity -> List of Lambda Identities that capture it.
|
||||
pub struct UpvalueAnalyzer;
|
||||
|
||||
impl UpvalueAnalyzer {
|
||||
pub fn analyze(root: &Node<UntypedKind>) -> HashSet<Identity> {
|
||||
let mut boxed = HashSet::new();
|
||||
pub fn analyze(root: &Node<UntypedKind>) -> HashMap<Identity, Vec<Identity>> {
|
||||
let mut capture_map: HashMap<Identity, HashSet<Identity>> = HashMap::new();
|
||||
let mut scopes = vec![HashMap::new()]; // Root scope
|
||||
Self::visit(root, &mut scopes, &mut boxed);
|
||||
boxed
|
||||
|
||||
Self::visit(root, &mut scopes, &mut capture_map, None);
|
||||
|
||||
// Convert HashSet back to sorted Vec for the final result
|
||||
capture_map.into_iter()
|
||||
.map(|(decl, lambdas)| (decl, lambdas.into_iter().collect()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn visit(node: &Node<UntypedKind>, scopes: &mut Vec<HashMap<Rc<str>, Identity>>, boxed: &mut HashSet<Identity>) {
|
||||
fn visit(
|
||||
node: &Node<UntypedKind>,
|
||||
scopes: &mut Vec<HashMap<Rc<str>, Identity>>,
|
||||
capture_map: &mut HashMap<Identity, HashSet<Identity>>,
|
||||
current_lambda: Option<Identity>
|
||||
) {
|
||||
match &node.kind {
|
||||
UntypedKind::Identifier(name) => {
|
||||
// Resolve name in scope stack (from inner to outer)
|
||||
for (depth, scope) in scopes.iter().rev().enumerate() {
|
||||
if let Some(id) = scope.get(name) {
|
||||
if let Some(decl_id) = scope.get(name) {
|
||||
if depth > 0 {
|
||||
// Captured from an outer scope!
|
||||
boxed.insert(id.clone());
|
||||
if let Some(lambda_id) = ¤t_lambda {
|
||||
capture_map.entry(decl_id.clone())
|
||||
.or_insert_with(HashSet::new)
|
||||
.insert(lambda_id.clone());
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
UntypedKind::Def { name, value } => {
|
||||
// 1. Visit initializer first (it executes in current scope)
|
||||
Self::visit(value, scopes, boxed);
|
||||
|
||||
// 2. Define the variable in current scope
|
||||
Self::visit(value, scopes, capture_map, current_lambda.clone());
|
||||
if let Some(current) = scopes.last_mut() {
|
||||
current.insert(name.clone(), node.identity.clone());
|
||||
}
|
||||
}
|
||||
UntypedKind::Lambda { params, body } => {
|
||||
// Enter new lambda scope
|
||||
let mut new_scope = HashMap::new();
|
||||
for p in params {
|
||||
// Parameters are defined in the new scope, masking outer ones.
|
||||
// We use the lambda's identity as a placeholder for parameter origin.
|
||||
new_scope.insert(p.clone(), node.identity.clone());
|
||||
}
|
||||
|
||||
scopes.push(new_scope);
|
||||
Self::visit(body, scopes, boxed);
|
||||
// The current node is the lambda causing captures in its body
|
||||
Self::visit(body, scopes, capture_map, Some(node.identity.clone()));
|
||||
scopes.pop();
|
||||
}
|
||||
UntypedKind::If { cond, then_br, else_br } => {
|
||||
Self::visit(cond, scopes, boxed);
|
||||
Self::visit(then_br, scopes, boxed);
|
||||
Self::visit(cond, scopes, capture_map, current_lambda.clone());
|
||||
Self::visit(then_br, scopes, capture_map, current_lambda.clone());
|
||||
if let Some(e) = else_br {
|
||||
Self::visit(e, scopes, boxed);
|
||||
Self::visit(e, scopes, capture_map, current_lambda.clone());
|
||||
}
|
||||
}
|
||||
UntypedKind::Assign { target, value } => {
|
||||
Self::visit(target, scopes, boxed);
|
||||
Self::visit(value, scopes, boxed);
|
||||
Self::visit(target, scopes, capture_map, current_lambda.clone());
|
||||
Self::visit(value, scopes, capture_map, current_lambda.clone());
|
||||
}
|
||||
UntypedKind::Call { callee, args } => {
|
||||
Self::visit(callee, scopes, boxed);
|
||||
Self::visit(callee, scopes, capture_map, current_lambda.clone());
|
||||
for arg in args {
|
||||
Self::visit(arg, scopes, boxed);
|
||||
Self::visit(arg, scopes, capture_map, current_lambda.clone());
|
||||
}
|
||||
}
|
||||
UntypedKind::Block { exprs } => {
|
||||
for expr in exprs {
|
||||
Self::visit(expr, scopes, boxed);
|
||||
Self::visit(expr, scopes, capture_map, current_lambda.clone());
|
||||
}
|
||||
}
|
||||
UntypedKind::Tuple { elements } => {
|
||||
for el in elements {
|
||||
Self::visit(el, scopes, boxed);
|
||||
Self::visit(el, scopes, capture_map, current_lambda.clone());
|
||||
}
|
||||
}
|
||||
UntypedKind::Map { entries } => {
|
||||
for (k, v) in entries {
|
||||
Self::visit(k, scopes, boxed);
|
||||
Self::visit(v, scopes, boxed);
|
||||
Self::visit(k, scopes, capture_map, current_lambda.clone());
|
||||
Self::visit(v, scopes, capture_map, current_lambda.clone());
|
||||
}
|
||||
}
|
||||
UntypedKind::Nop | UntypedKind::Constant(_) | UntypedKind::Extension(_) => {}
|
||||
|
||||
@@ -8,6 +8,8 @@ use crate::ast::vm::VM;
|
||||
|
||||
use crate::ast::compiler::tco::TCO;
|
||||
|
||||
use crate::ast::compiler::dumper::Dumper;
|
||||
|
||||
pub struct Environment {
|
||||
pub global_names: Rc<RefCell<HashMap<String, (u32, StaticType)>>>,
|
||||
pub global_values: Rc<RefCell<Vec<Value>>>,
|
||||
@@ -105,6 +107,16 @@ impl Environment {
|
||||
});
|
||||
}
|
||||
|
||||
pub fn dump_ast(&self, source: &str) -> Result<String, String> {
|
||||
let mut parser = Parser::new(source)?;
|
||||
let untyped_ast = parser.parse_expression()?;
|
||||
|
||||
let bound_ast = Binder::bind_root(self.global_names.clone(), &untyped_ast)?;
|
||||
let optimized_ast = TCO::optimize(bound_ast);
|
||||
|
||||
Ok(Dumper::dump(&optimized_ast))
|
||||
}
|
||||
|
||||
pub fn run_script(&self, source: &str) -> Result<Value, String> {
|
||||
// 1. Parse
|
||||
let mut parser = Parser::new(source)?;
|
||||
|
||||
+2
-2
@@ -82,9 +82,9 @@ impl VM {
|
||||
Ok(val)
|
||||
},
|
||||
|
||||
BoundKind::DefLocal { slot, value, is_boxed } => {
|
||||
BoundKind::DefLocal { slot, value, captured_by } => {
|
||||
let val = self.eval(value)?;
|
||||
let final_val = if *is_boxed {
|
||||
let final_val = if !captured_by.is_empty() {
|
||||
Value::Cell(Rc::new(RefCell::new(val)))
|
||||
} else {
|
||||
val
|
||||
|
||||
+19
-1
@@ -85,7 +85,21 @@ impl CompilerApp {
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
self.output_log = format!("Error: {}", e);
|
||||
self.output_log = format!("Error during Compilation/Execution: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn dump_ast(&mut self) {
|
||||
match self.env.dump_ast(&self.source_code) {
|
||||
Ok(dump) => {
|
||||
self.output_log = format!(
|
||||
"--- BOUND AST DUMP ---\n\n{}",
|
||||
dump
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
self.output_log = format!("Error during AST Binding: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -213,6 +227,10 @@ impl eframe::App for CompilerApp {
|
||||
self.compile();
|
||||
}
|
||||
|
||||
if ui.button("Dump AST").clicked() {
|
||||
self.dump_ast();
|
||||
}
|
||||
|
||||
if let Some(path) = &self.current_example_path {
|
||||
let file_name = path.file_name().unwrap_or_default().to_string_lossy();
|
||||
if ui.button(format!("Save {} (Ctrl+S)", file_name)).clicked() {
|
||||
|
||||
Reference in New Issue
Block a user