Add AST dumper and improve upvalue analysis

The `UpvalueAnalyzer` now correctly identifies which lambdas capture
which variable declarations, rather than just marking declarations that
need boxing. This information is stored in a `capture_map`.

The `Binder` has been updated to use this new `capture_map` instead of a
`HashSet` of boxed declarations. The `DefLocal` node in `bound_nodes.rs`
now stores a `captured_by` field, which is a list of lambda identities
that capture the local variable.

A new `dumper` module has been added to provide a human-readable
representation of the bound AST, including information about captured
variables.

The `VM` has been updated to use the `captured_by` field to determine if
a local variable needs to be stored in a `Cell`, rather than relying on
a boolean `is_boxed` flag.

An example script `extreme_capture.myc` has been added to test deep
nesting and variable capture.

A new "Dump AST" button has been added to the UI, which uses the new
`Dumper` to display the bound AST.
This commit is contained in:
Michael Schimmel
2026-02-18 00:35:08 +01:00
parent 4d9adccab5
commit 25e3bc1d01
10 changed files with 269 additions and 49 deletions
+25
View File
@@ -0,0 +1,25 @@
;; Output: 36
(do
;; Excessive capture test
;; This script creates deep nesting where the inner-most lambda
;; grabs variables from every single outer scope.
(def excessive (fn [v1]
(do
(def v2 2)
(fn [v3]
(do
(def v4 4)
(fn [v5]
(do
(def v6 6)
(fn [v7]
(do
(def v8 8)
;; v1, v3, v5, v7 are parameters (captured as upvalues)
;; v2, v4, v6, v8 are locals (captured from outer scopes)
(+ v1 v2 v3 v4 v5 v6 v7 v8))))))))))
;; Execution: (1 + 2 + 3 + 4 + 5 + 6 + 7 + 8) = 36
((((excessive 1) 3) 5) 7)
)
+14 -14
View File
@@ -1,4 +1,4 @@
use std::collections::{HashMap, BTreeMap, HashSet}; use std::collections::{HashMap, BTreeMap};
use std::rc::Rc; use std::rc::Rc;
use std::cell::RefCell; use std::cell::RefCell;
use crate::ast::nodes::{Node, UntypedKind}; use crate::ast::nodes::{Node, UntypedKind};
@@ -64,28 +64,28 @@ pub struct Binder {
functions: Vec<FunctionCompiler>, functions: Vec<FunctionCompiler>,
// Globals mapping: Name -> (Index, Type) // Globals mapping: Name -> (Index, Type)
globals: Rc<RefCell<HashMap<String, (u32, StaticType)>>>, globals: Rc<RefCell<HashMap<String, (u32, StaticType)>>>,
// Identities of definitions that need boxing (pre-calculated) // Map of Declaration Identity -> List of Lambda Identities that capture it
boxed_declarations: HashSet<Identity>, capture_map: HashMap<Identity, Vec<Identity>>,
} }
impl Binder { impl Binder {
pub fn new(globals: Rc<RefCell<HashMap<String, (u32, StaticType)>>>) -> Self { pub fn new(globals: Rc<RefCell<HashMap<String, (u32, StaticType)>>>) -> Self {
Self::with_boxed(globals, HashSet::new()) Self::with_boxed(globals, HashMap::new())
} }
fn with_boxed(globals: Rc<RefCell<HashMap<String, (u32, StaticType)>>>, boxed: HashSet<Identity>) -> Self { fn with_boxed(globals: Rc<RefCell<HashMap<String, (u32, StaticType)>>>, captures: HashMap<Identity, Vec<Identity>>) -> Self {
let mut binder = Self { let mut binder = Self {
functions: Vec::new(), functions: Vec::new(),
globals, globals,
boxed_declarations: boxed, capture_map: captures,
}; };
binder.functions.push(FunctionCompiler::new()); binder.functions.push(FunctionCompiler::new());
binder binder
} }
pub fn bind_root(globals: Rc<RefCell<HashMap<String, (u32, StaticType)>>>, node: &Node<UntypedKind>) -> Result<Node<BoundKind, StaticType>, String> { pub fn bind_root(globals: Rc<RefCell<HashMap<String, (u32, StaticType)>>>, node: &Node<UntypedKind>) -> Result<Node<BoundKind, StaticType>, String> {
let boxed = crate::ast::compiler::upvalues::UpvalueAnalyzer::analyze(node); let captures = crate::ast::compiler::upvalues::UpvalueAnalyzer::analyze(node);
let mut binder = Self::with_boxed(globals, boxed); let mut binder = Self::with_boxed(globals, captures);
binder.bind(node) binder.bind(node)
} }
@@ -170,11 +170,11 @@ impl Binder {
info.ty = ty.clone(); info.ty = ty.clone();
} }
} }
let is_boxed = self.boxed_declarations.contains(&node.identity); let captured_by = self.capture_map.get(&node.identity).cloned().unwrap_or_default();
Ok(self.make_node(node.identity.clone(), BoundKind::DefLocal { Ok(self.make_node(node.identity.clone(), BoundKind::DefLocal {
slot: slot_or_idx, slot: slot_or_idx,
value: Box::new(val_node) , value: Box::new(val_node) ,
is_boxed captured_by
}, ty)) }, ty))
} }
}, },
@@ -361,8 +361,8 @@ mod tests {
if let BoundKind::Lambda { body, .. } = &bound.kind { if let BoundKind::Lambda { body, .. } = &bound.kind {
if let BoundKind::Block { exprs } = &body.kind { if let BoundKind::Block { exprs } = &body.kind {
let x_decl = &exprs[0]; let x_decl = &exprs[0];
if let BoundKind::DefLocal { is_boxed, .. } = &x_decl.kind { if let BoundKind::DefLocal { captured_by, .. } = &x_decl.kind {
assert!(is_boxed, "Variable 'x' should be marked as boxed because it is captured by lambda 'f'"); assert!(!captured_by.is_empty(), "Variable 'x' should have capturers because it is used in lambda 'f'");
} else { } else {
panic!("First expression in block should be DefLocal, got {:?}", x_decl.kind); panic!("First expression in block should be DefLocal, got {:?}", x_decl.kind);
} }
@@ -386,8 +386,8 @@ mod tests {
if let BoundKind::Lambda { body, .. } = &bound.kind { if let BoundKind::Lambda { body, .. } = &bound.kind {
if let BoundKind::Block { exprs } = &body.kind { if let BoundKind::Block { exprs } = &body.kind {
let x_decl = &exprs[0]; let x_decl = &exprs[0];
if let BoundKind::DefLocal { is_boxed, .. } = &x_decl.kind { if let BoundKind::DefLocal { captured_by, .. } = &x_decl.kind {
assert!(!is_boxed, "Variable 'x' should NOT be boxed as it is not captured"); assert!(captured_by.is_empty(), "Variable 'x' should NOT have any capturers");
} else { } else {
panic!("First expression should be DefLocal"); panic!("First expression should be DefLocal");
} }
+2 -2
View File
@@ -1,5 +1,5 @@
use std::rc::Rc; use std::rc::Rc;
use crate::ast::types::{Value, StaticType}; use crate::ast::types::{Value, StaticType, Identity};
use crate::ast::nodes::Node; use crate::ast::nodes::Node;
#[derive(Debug, Clone, Copy, PartialEq)] #[derive(Debug, Clone, Copy, PartialEq)]
@@ -27,7 +27,7 @@ pub enum BoundKind {
DefLocal { DefLocal {
slot: u32, slot: u32,
value: Box<Node<BoundKind, StaticType>>, value: Box<Node<BoundKind, StaticType>>,
is_boxed: bool, captured_by: Vec<Identity>,
}, },
If { If {
+154
View File
@@ -0,0 +1,154 @@
use crate::ast::nodes::Node;
use crate::ast::compiler::bound_nodes::BoundKind;
use crate::ast::types::StaticType;
/// Human-readable AST dumper for the bound AST.
pub struct Dumper {
output: String,
indent: usize,
}
impl Dumper {
/// Produces a formatted string representation of the given bound AST node and its children.
pub fn dump(node: &Node<BoundKind, StaticType>) -> String {
let mut dumper = Self {
output: String::new(),
indent: 0,
};
dumper.visit(node);
dumper.output
}
fn write_indent(&mut self) {
for _ in 0..self.indent {
self.output.push_str(" ");
}
}
fn log(&mut self, label: &str, node: &Node<BoundKind, StaticType>) {
self.write_indent();
self.output.push_str(label);
self.output.push_str(&format!(" <Type: {:?}>\n", node.ty));
}
fn visit(&mut self, node: &Node<BoundKind, StaticType>) {
match &node.kind {
BoundKind::Nop => self.log("Nop", node),
BoundKind::Constant(v) => self.log(&format!("Constant: {}", v), node),
BoundKind::Get(addr) => self.log(&format!("Get: {:?}", addr), node),
BoundKind::Set { addr, value } => {
self.log(&format!("Set: {:?}", addr), node);
self.indent += 1;
self.visit(value);
self.indent -= 1;
}
BoundKind::DefLocal { slot, value, captured_by } => {
let capture_info = if captured_by.is_empty() {
String::from("not captured")
} else {
format!("captured by {} lambdas", captured_by.len())
};
self.log(&format!("DefLocal (Slot: {}, {})", slot, capture_info), node);
self.indent += 1;
if !captured_by.is_empty() {
for capturer in captured_by {
self.write_indent();
self.output.push_str(&format!("- Capturer: Lambda at line {}, col {}\n",
capturer.location.line, capturer.location.col));
}
}
self.visit(value);
self.indent -= 1;
}
BoundKind::DefGlobal { global_index, value } => {
self.log(&format!("DefGlobal (Index: {})", global_index), node);
self.indent += 1;
self.visit(value);
self.indent -= 1;
}
BoundKind::If { cond, then_br, else_br } => {
self.log("If", node);
self.indent += 1;
self.write_indent();
self.output.push_str("Condition:\n");
self.visit(cond);
self.write_indent();
self.output.push_str("Then:\n");
self.visit(then_br);
if let Some(e) = else_br {
self.write_indent();
self.output.push_str("Else:\n");
self.visit(e);
}
self.indent -= 1;
}
BoundKind::Lambda { param_count, upvalues, body } => {
self.log(&format!("Lambda (Params: {}, Upvalues: {})", param_count, upvalues.len()), node);
self.indent += 1;
if !upvalues.is_empty() {
self.write_indent();
self.output.push_str(&format!("Upvalues: {:?}\n", upvalues));
}
self.visit(body);
self.indent -= 1;
}
BoundKind::Call { callee, args } => {
self.log("Call", node);
self.indent += 1;
self.visit(callee);
for arg in args {
self.visit(arg);
}
self.indent -= 1;
}
BoundKind::TailCall { callee, args } => {
self.log("TailCall (TCO)", node);
self.indent += 1;
self.visit(callee);
for arg in args {
self.visit(arg);
}
self.indent -= 1;
}
BoundKind::Block { exprs } => {
self.log("Block", node);
self.indent += 1;
for expr in exprs {
self.visit(expr);
}
self.indent -= 1;
}
BoundKind::Tuple { elements } => {
self.log("Tuple", node);
self.indent += 1;
for el in elements {
self.visit(el);
}
self.indent -= 1;
}
BoundKind::Map { entries } => {
self.log("Map", node);
self.indent += 1;
for (k, v) in entries {
self.visit(k);
self.visit(v);
}
self.indent -= 1;
}
}
}
}
+2
View File
@@ -2,8 +2,10 @@ pub mod binder;
pub mod bound_nodes; pub mod bound_nodes;
pub mod tco; pub mod tco;
pub mod upvalues; pub mod upvalues;
pub mod dumper;
pub use binder::*; pub use binder::*;
pub use bound_nodes::*; pub use bound_nodes::*;
pub use tco::*; pub use tco::*;
pub use upvalues::*; pub use upvalues::*;
pub use dumper::*;
+2 -2
View File
@@ -98,9 +98,9 @@ impl TCO {
..node ..node
} }
}, },
BoundKind::DefLocal { slot, value, is_boxed } => { BoundKind::DefLocal { slot, value, captured_by } => {
Node { Node {
kind: BoundKind::DefLocal { slot, value: Box::new(Self::transform(*value, false)), is_boxed }, kind: BoundKind::DefLocal { slot, value: Box::new(Self::transform(*value, false)), captured_by },
..node ..node
} }
}, },
+37 -28
View File
@@ -3,85 +3,94 @@ use std::rc::Rc;
use crate::ast::nodes::{Node, UntypedKind}; use crate::ast::nodes::{Node, UntypedKind};
use crate::ast::types::Identity; use crate::ast::types::Identity;
/// Analyzes the AST to find all variable declarations that are captured by nested lambdas. /// Analyzes the AST to find which lambdas capture which variable declarations.
/// Returns a set of Identities corresponding to the 'Def' nodes that need boxing. /// Returns a map: Declaration Identity -> List of Lambda Identities that capture it.
pub struct UpvalueAnalyzer; pub struct UpvalueAnalyzer;
impl UpvalueAnalyzer { impl UpvalueAnalyzer {
pub fn analyze(root: &Node<UntypedKind>) -> HashSet<Identity> { pub fn analyze(root: &Node<UntypedKind>) -> HashMap<Identity, Vec<Identity>> {
let mut boxed = HashSet::new(); let mut capture_map: HashMap<Identity, HashSet<Identity>> = HashMap::new();
let mut scopes = vec![HashMap::new()]; // Root scope let mut scopes = vec![HashMap::new()]; // Root scope
Self::visit(root, &mut scopes, &mut boxed);
boxed Self::visit(root, &mut scopes, &mut capture_map, None);
// Convert HashSet back to sorted Vec for the final result
capture_map.into_iter()
.map(|(decl, lambdas)| (decl, lambdas.into_iter().collect()))
.collect()
} }
fn visit(node: &Node<UntypedKind>, scopes: &mut Vec<HashMap<Rc<str>, Identity>>, boxed: &mut HashSet<Identity>) { fn visit(
node: &Node<UntypedKind>,
scopes: &mut Vec<HashMap<Rc<str>, Identity>>,
capture_map: &mut HashMap<Identity, HashSet<Identity>>,
current_lambda: Option<Identity>
) {
match &node.kind { match &node.kind {
UntypedKind::Identifier(name) => { UntypedKind::Identifier(name) => {
// Resolve name in scope stack (from inner to outer) // Resolve name in scope stack (from inner to outer)
for (depth, scope) in scopes.iter().rev().enumerate() { for (depth, scope) in scopes.iter().rev().enumerate() {
if let Some(id) = scope.get(name) { if let Some(decl_id) = scope.get(name) {
if depth > 0 { if depth > 0 {
// Captured from an outer scope! // Captured from an outer scope!
boxed.insert(id.clone()); if let Some(lambda_id) = &current_lambda {
capture_map.entry(decl_id.clone())
.or_insert_with(HashSet::new)
.insert(lambda_id.clone());
}
} }
break; break;
} }
} }
} }
UntypedKind::Def { name, value } => { UntypedKind::Def { name, value } => {
// 1. Visit initializer first (it executes in current scope) Self::visit(value, scopes, capture_map, current_lambda.clone());
Self::visit(value, scopes, boxed);
// 2. Define the variable in current scope
if let Some(current) = scopes.last_mut() { if let Some(current) = scopes.last_mut() {
current.insert(name.clone(), node.identity.clone()); current.insert(name.clone(), node.identity.clone());
} }
} }
UntypedKind::Lambda { params, body } => { UntypedKind::Lambda { params, body } => {
// Enter new lambda scope
let mut new_scope = HashMap::new(); let mut new_scope = HashMap::new();
for p in params { for p in params {
// Parameters are defined in the new scope, masking outer ones.
// We use the lambda's identity as a placeholder for parameter origin.
new_scope.insert(p.clone(), node.identity.clone()); new_scope.insert(p.clone(), node.identity.clone());
} }
scopes.push(new_scope); scopes.push(new_scope);
Self::visit(body, scopes, boxed); // The current node is the lambda causing captures in its body
Self::visit(body, scopes, capture_map, Some(node.identity.clone()));
scopes.pop(); scopes.pop();
} }
UntypedKind::If { cond, then_br, else_br } => { UntypedKind::If { cond, then_br, else_br } => {
Self::visit(cond, scopes, boxed); Self::visit(cond, scopes, capture_map, current_lambda.clone());
Self::visit(then_br, scopes, boxed); Self::visit(then_br, scopes, capture_map, current_lambda.clone());
if let Some(e) = else_br { if let Some(e) = else_br {
Self::visit(e, scopes, boxed); Self::visit(e, scopes, capture_map, current_lambda.clone());
} }
} }
UntypedKind::Assign { target, value } => { UntypedKind::Assign { target, value } => {
Self::visit(target, scopes, boxed); Self::visit(target, scopes, capture_map, current_lambda.clone());
Self::visit(value, scopes, boxed); Self::visit(value, scopes, capture_map, current_lambda.clone());
} }
UntypedKind::Call { callee, args } => { UntypedKind::Call { callee, args } => {
Self::visit(callee, scopes, boxed); Self::visit(callee, scopes, capture_map, current_lambda.clone());
for arg in args { for arg in args {
Self::visit(arg, scopes, boxed); Self::visit(arg, scopes, capture_map, current_lambda.clone());
} }
} }
UntypedKind::Block { exprs } => { UntypedKind::Block { exprs } => {
for expr in exprs { for expr in exprs {
Self::visit(expr, scopes, boxed); Self::visit(expr, scopes, capture_map, current_lambda.clone());
} }
} }
UntypedKind::Tuple { elements } => { UntypedKind::Tuple { elements } => {
for el in elements { for el in elements {
Self::visit(el, scopes, boxed); Self::visit(el, scopes, capture_map, current_lambda.clone());
} }
} }
UntypedKind::Map { entries } => { UntypedKind::Map { entries } => {
for (k, v) in entries { for (k, v) in entries {
Self::visit(k, scopes, boxed); Self::visit(k, scopes, capture_map, current_lambda.clone());
Self::visit(v, scopes, boxed); Self::visit(v, scopes, capture_map, current_lambda.clone());
} }
} }
UntypedKind::Nop | UntypedKind::Constant(_) | UntypedKind::Extension(_) => {} UntypedKind::Nop | UntypedKind::Constant(_) | UntypedKind::Extension(_) => {}
+12
View File
@@ -8,6 +8,8 @@ use crate::ast::vm::VM;
use crate::ast::compiler::tco::TCO; use crate::ast::compiler::tco::TCO;
use crate::ast::compiler::dumper::Dumper;
pub struct Environment { pub struct Environment {
pub global_names: Rc<RefCell<HashMap<String, (u32, StaticType)>>>, pub global_names: Rc<RefCell<HashMap<String, (u32, StaticType)>>>,
pub global_values: Rc<RefCell<Vec<Value>>>, pub global_values: Rc<RefCell<Vec<Value>>>,
@@ -105,6 +107,16 @@ impl Environment {
}); });
} }
pub fn dump_ast(&self, source: &str) -> Result<String, String> {
let mut parser = Parser::new(source)?;
let untyped_ast = parser.parse_expression()?;
let bound_ast = Binder::bind_root(self.global_names.clone(), &untyped_ast)?;
let optimized_ast = TCO::optimize(bound_ast);
Ok(Dumper::dump(&optimized_ast))
}
pub fn run_script(&self, source: &str) -> Result<Value, String> { pub fn run_script(&self, source: &str) -> Result<Value, String> {
// 1. Parse // 1. Parse
let mut parser = Parser::new(source)?; let mut parser = Parser::new(source)?;
+2 -2
View File
@@ -82,9 +82,9 @@ impl VM {
Ok(val) Ok(val)
}, },
BoundKind::DefLocal { slot, value, is_boxed } => { BoundKind::DefLocal { slot, value, captured_by } => {
let val = self.eval(value)?; let val = self.eval(value)?;
let final_val = if *is_boxed { let final_val = if !captured_by.is_empty() {
Value::Cell(Rc::new(RefCell::new(val))) Value::Cell(Rc::new(RefCell::new(val)))
} else { } else {
val val
+19 -1
View File
@@ -85,7 +85,21 @@ impl CompilerApp {
); );
} }
Err(e) => { Err(e) => {
self.output_log = format!("Error: {}", e); self.output_log = format!("Error during Compilation/Execution: {}", e);
}
}
}
fn dump_ast(&mut self) {
match self.env.dump_ast(&self.source_code) {
Ok(dump) => {
self.output_log = format!(
"--- BOUND AST DUMP ---\n\n{}",
dump
);
}
Err(e) => {
self.output_log = format!("Error during AST Binding: {}", e);
} }
} }
} }
@@ -213,6 +227,10 @@ impl eframe::App for CompilerApp {
self.compile(); self.compile();
} }
if ui.button("Dump AST").clicked() {
self.dump_ast();
}
if let Some(path) = &self.current_example_path { if let Some(path) = &self.current_example_path {
let file_name = path.file_name().unwrap_or_default().to_string_lossy(); let file_name = path.file_name().unwrap_or_default().to_string_lossy();
if ui.button(format!("Save {} (Ctrl+S)", file_name)).clicked() { if ui.button(format!("Save {} (Ctrl+S)", file_name)).clicked() {