use crate::ast::compiler::bound_nodes::{Address, AnalyzedNode, BoundKind}; use crate::ast::compiler::tco::ExecNode; use crate::ast::nodes::Node; use crate::ast::types::{Object, Value}; use std::any::Any; use std::cell::RefCell; use std::rc::Rc; #[derive(Debug, Clone)] pub struct Closure { /// The analyzed parameter pattern. pub parameter_node: Rc, /// The analyzed body (before TCO). pub function_node: Rc, /// The executable node (after TCO). pub exec_node: Rc, pub upvalues: Vec>>, pub positional_count: Option, } impl Closure { #[inline] pub fn new( params: Rc, body: Rc, exec: Rc, upvalues: Vec>>, positional_count: Option, ) -> Self { Self { parameter_node: params, function_node: body, exec_node: exec, upvalues, positional_count, } } } impl Object for Closure { fn type_name(&self) -> &'static str { "closure" } fn as_any(&self) -> &dyn Any { self } } #[derive(Debug)] struct CallFrame { stack_base: usize, closure: Option>, } pub trait VMObserver { const ACTIVE: bool = false; fn before_eval(&mut self, _vm: &VM, _node: &ExecNode) {} fn after_eval(&mut self, _vm: &VM, _node: &ExecNode, _res: &Result) {} } pub struct NoOpObserver; impl VMObserver for NoOpObserver {} pub struct TracingObserver { pub logs: Vec, indent: usize, } impl TracingObserver { pub fn new() -> Self { Self { logs: Vec::new(), indent: 0, } } fn pad(&self) -> String { "| ".repeat(self.indent) } } impl Default for TracingObserver { fn default() -> Self { Self::new() } } impl VMObserver for TracingObserver { const ACTIVE: bool = true; fn before_eval(&mut self, _vm: &VM, node: &ExecNode) { let pad = self.pad(); let metrics = &node.ty.original.ty; self.logs.push(format!( "{}{} [{} | P:{:?}{}]: {{", pad, node.kind.display_name(), node.ty.ty, metrics.purity, if metrics.is_recursive { " | REC" } else { "" } )); self.indent += 1; } fn after_eval(&mut self, vm: &VM, node: &ExecNode, res: &Result) { self.indent = self.indent.saturating_sub(1); let pad = self.pad(); match &node.kind { BoundKind::Define { .. } | BoundKind::Set { .. } => { let s_pad = format!("{}| ", pad); self.logs.push(format!("{}--- Scope Status ---", s_pad)); self.logs.push(format!( "{}Stack (top 5): {:?}", s_pad, vm.stack.iter().rev().take(5).collect::>() )); } _ => {} } let res_str = match res { Ok(v) => format!("{}", v), Err(e) => format!("ERROR: {}", e), }; self.logs.push(format!("{}}} -> {}", pad, res_str)); } } pub struct VM { stack: Vec, globals: Rc>>, frames: Vec, } impl VM { pub fn new(globals: Rc>>) -> Self { Self { stack: Vec::new(), globals, frames: Vec::new(), } } pub fn run(&mut self, root: &ExecNode) -> Result { self.stack.clear(); self.frames.clear(); self.frames.push(CallFrame { stack_base: 0, closure: None, }); let result = self.eval(root); self.frames.pop(); self.resolve_tail_calls(&mut NoOpObserver, result) } pub fn resolve_tail_calls( &mut self, observer: &mut O, mut result: Result, ) -> Result { loop { match result { Ok(Value::TailCallRequest(payload)) => { let (next_obj, next_args) = *payload; if let Some(closure) = next_obj.as_any().downcast_ref::() { self.stack.clear(); // frames should be empty here since we popped before entering the loop self.frames.push(CallFrame { stack_base: 0, closure: Some(next_obj.clone()), }); if let Some(count) = closure.positional_count && next_args.len() == count as usize { self.stack.extend(next_args); } else { self.unpack(&closure.parameter_node, &next_args, &mut 0)?; } result = self.eval_observed(observer, &closure.exec_node); self.frames.pop(); } else if let Some(series) = next_obj.as_series() { if next_args.len() != 1 { return Err(format!( "{} indexer expects exactly 1 argument (the lookback index), got {}", next_obj.type_name(), next_args.len() )); } if let Value::Int(idx) = &next_args[0] { if *idx < 0 { return Err(format!( "{} lookback index cannot be negative: {}", next_obj.type_name(), idx )); } result = Ok(series.get_item(*idx as usize).unwrap_or(Value::Void)); } else { return Err(format!( "{} index must be an integer, got {}", next_obj.type_name(), next_args[0] )); } } else { return Err(format!( "Tail call target is not callable: {}", next_obj.type_name() )); } } _ => return result, } } } pub fn run_with_args( &mut self, closure_obj: Rc, args: &[Value], ) -> Result { let closure = closure_obj.as_any().downcast_ref::().unwrap(); self.stack.clear(); self.frames.clear(); self.frames.push(CallFrame { stack_base: 0, closure: Some(closure_obj.clone()), }); if let Some(count) = closure.positional_count && args.len() == count as usize { self.stack.extend_from_slice(args); } else { self.unpack(&closure.parameter_node, args, &mut 0)?; } self.eval(&closure.exec_node) } pub fn run_with_args_observed( &mut self, observer: &mut O, closure_obj: Rc, args: &[Value], ) -> Result { let closure = closure_obj.as_any().downcast_ref::().unwrap(); self.stack.clear(); self.frames.clear(); self.frames.push(CallFrame { stack_base: 0, closure: Some(closure_obj.clone()), }); if let Some(count) = closure.positional_count && args.len() == count as usize { self.stack.extend_from_slice(args); } else { self.unpack(&closure.parameter_node, args, &mut 0)?; } self.eval_observed(observer, &closure.exec_node) } pub fn run_with_observer( &mut self, observer: &mut O, root: &ExecNode, ) -> Result { self.stack.clear(); self.frames.clear(); self.frames.push(CallFrame { stack_base: 0, closure: None, }); let result = self.eval_observed(observer, root); self.frames.pop(); self.resolve_tail_calls(observer, result) } #[inline(always)] fn eval(&mut self, node: &ExecNode) -> Result { self.eval_internal(&mut NoOpObserver, node) } fn eval_observed( &mut self, observer: &mut O, node: &ExecNode, ) -> Result { self.eval_internal(observer, node) } #[inline(always)] fn eval_internal( &mut self, obs: &mut O, node: &ExecNode, ) -> Result { if O::ACTIVE { obs.before_eval(self, node); let result = self.eval_core(obs, node); obs.after_eval(self, node, &result); result } else { self.eval_core(obs, node) } } #[inline(always)] fn eval_core(&mut self, obs: &mut O, node: &ExecNode) -> Result { match &node.kind { BoundKind::Nop => Ok(Value::Void), BoundKind::Constant(v) => Ok(v.clone()), BoundKind::Define { addr, value, captured_by, .. } => { let val = self.eval_internal(obs, value)?; let final_val = if !captured_by.is_empty() && matches!(addr, Address::Local(_)) { Value::Cell(Rc::new(RefCell::new(val))) } else { val }; self.set_value(*addr, final_val.clone())?; Ok(final_val) } BoundKind::Destructure { pattern, value } => { let val = self.eval_internal(obs, value)?; let mut offset = 0; // Destructuring works on tuples/vectors, or single values wrapped in a slice if let Some(vals) = val.as_slice() { self.unpack(pattern, vals, &mut offset)?; } else { self.unpack(pattern, std::slice::from_ref(&val), &mut offset)?; } Ok(val) } BoundKind::Get { addr, .. } => self.get_value(*addr), BoundKind::FieldAccessor(k) => Ok(Value::FieldAccessor(*k)), BoundKind::GetField { rec, field } => { let rec_val = self.eval_internal(obs, rec)?; // In Rust, pattern matching (`match`) is the idiomatic way to handle variants safely. // Previously, this only handled `Value::Record`. Now, we handle objects (like `RecordSeries`) polymorphically. match rec_val { // Case 1: The classic Record. // This is a struct-like tuple containing an Arc and a Vec. Value::Record(layout, values) => { if let Some(idx) = layout.index_of(*field) { Ok(values[idx].clone()) } else { Err(format!("Record does not have field :{}", field.name())) } } // Case 2: A dynamic Object (our SoA / Struct-of-Arrays optimization). // `Value::Object` holds an `Rc` - a reference-counted trait object (type-erased). Value::Object(obj) => { // 1. We get the raw `&dyn Any` reference (Rust's standard mechanism for runtime type reflection). let any_ptr = obj.as_any(); // 2. Downcast! We check at runtime if the pointer actually points to a `RecordSeries`. // `downcast_ref` is very fast (essentially an O(1) type ID comparison under the hood). if let Some(record_series) = any_ptr.downcast_ref::() { // 3. We call our highly performant 0-copy method on the series. // It returns an `Rc>`, which is a shared pointer // to the concrete column array (e.g., a `ScalarSeries`). if let Some(field_series) = record_series.field(*field) { // 4. We wrap this RefCell in our `SeriesView` struct. // The `SeriesView` acts as a pure `Object` for the VM, holding the reference. // CRITICAL: No array elements are copied here! This is pure, fast pointer juggling. // This single operation turns a SoA `RecordSeries` into a high-speed `FloatSeries` view. let view = crate::ast::rtl::series::SeriesView::new(field_series, *field); return Ok(Value::Object(std::rc::Rc::new(view))); } else { return Err(format!( "RecordSeries does not have field :{}", field.name() )); } } // Fallback if it's another type of object that is not a RecordSeries. Err(format!( "Attempt to access field on non-record object: {}", obj.type_name() )) } // Error handling for primitives (Int, Float, etc.). _ => Err(format!( "Attempt to access field on non-record: {}", rec_val )), } } BoundKind::Set { addr, value } => { let val = self.eval_internal(obs, value)?; self.set_value(*addr, val.clone())?; Ok(val) } BoundKind::If { cond, then_br, else_br, } => { let c = self.eval_internal(obs, cond)?; if c.is_truthy() { self.eval_internal(obs, then_br) } else if let Some(e) = else_br { self.eval_internal(obs, e) } else { Ok(Value::Void) } } BoundKind::Pipe { inputs, lambda, out_type, } => { use crate::ast::rtl::streams::{PipelineNode, StreamNode}; let mut obs_streams = Vec::new(); for input in inputs { let val = self.eval_internal(obs, input)?; if let Value::Object(obj) = val { if let Some(s) = obj.as_any().downcast_ref::() { obs_streams.push(s.inner.clone()); } else if let Some(p) = obj.as_any().downcast_ref::() { obs_streams.push(p.stream.clone()); } else { return Err(format!( "Pipe input must be a stream, found {}", obj.type_name() )); } } else { return Err("Pipe input must be an object (stream)".to_string()); } } let lambda_val = self.eval_internal(obs, lambda)?; let lambda_obj = if let Value::Object(obj) = lambda_val { obj } else { return Err("Pipe lambda must be a function/closure".to_string()); }; // Create the persistent execution closure for the PipeStream let mut pipe_vm = crate::ast::vm::VM::new(self.globals.clone()); let my_closure = lambda_obj.clone(); let executor: Box = Box::new(move |args: &[Value]| -> Value { match pipe_vm.run_with_args(my_closure.clone(), args) { Ok(res) => res, Err(e) => panic!("Pipeline lambda execution failed: {}", e), } }); // Delegate to the RTL Factory for specialized buffer instantiation let node = crate::ast::rtl::streams::build_pipeline_node(obs_streams, executor, out_type); Ok(Value::Object(node)) } BoundKind::Block { exprs } => { let mut last = Value::Void; for e in exprs { last = self.eval_internal(obs, e)?; } Ok(last) } BoundKind::Lambda { params, upvalues, body, positional_count, } => { let mut captured = Vec::with_capacity(upvalues.len()); for addr in upvalues { captured.push(self.capture_upvalue(*addr)?); } let closure = Closure::new( params.ty.original.clone(), body.ty.original.clone(), body.clone(), captured, *positional_count, ); Ok(Value::Object(Rc::new(closure))) } BoundKind::Call { callee, args } => { let func_val = self.eval_internal(obs, callee)?; let base = self.stack.len(); if let Err(e) = self.eval_args_to_stack(obs, args) { self.stack.truncate(base); return Err(e); } if node.ty.is_tail { let arg_vals = self.stack[base..].to_vec(); self.stack.truncate(base); match func_val { Value::Object(obj) => { return Ok(Value::TailCallRequest(Box::new((obj, arg_vals)))); } Value::Function(f) => return Ok((f.func)(&arg_vals)), Value::FieldAccessor(k) => { if arg_vals.len() != 1 { return Err(format!( "Field accessor .{} expects exactly 1 argument, got {}", k.name(), arg_vals.len() )); } let rec = &arg_vals[0]; if let Value::Record(layout, values) = rec { if let Some(idx) = layout.index_of(k) { return Ok(values[idx].clone()); } else { return Err(format!( "Record does not have field :{}", k.name() )); } } else if let Value::Object(obj) = rec { // Polymorphic Field Access: Allow `.field` on a RecordSeries if let Some(rs) = obj .as_any() .downcast_ref::() { if let Some(field_series) = rs.field(k) { let view = crate::ast::rtl::series::SeriesView::new( field_series, k, ); return Ok(Value::Object(std::rc::Rc::new(view))); } else { return Err(format!( "RecordSeries does not have field :{}", k.name() )); } } return Err(format!( "Field accessor .{} expects a record or RecordSeries, got {}", k.name(), obj.type_name() )); } else { return Err(format!( "Field accessor .{} expects a record or RecordSeries, got {}", k.name(), rec )); } } _ => { return Err(format!( "Tail call target is not a function: {}", func_val )); } } } // Standard Call Path let mut current_func = func_val; loop { let result = match ¤t_func { Value::Function(f) => { let res = (f.func)(&self.stack[base..]); self.stack.truncate(base); return Ok(res); } Value::FieldAccessor(k) => { let arg_len = self.stack.len() - base; let res = if arg_len != 1 { Err(format!( "Field accessor .{} expects exactly 1 argument, got {}", k.name(), arg_len )) } else { let rec = &self.stack[base]; if let Value::Record(layout, values) = rec { if let Some(idx) = layout.index_of(*k) { Ok(values[idx].clone()) } else { Err(format!("Record does not have field :{}", k.name())) } } else if let Value::Object(obj) = rec { if let Some(rs) = obj .as_any() .downcast_ref::() { if let Some(field_series) = rs.field(*k) { let view = crate::ast::rtl::series::SeriesView::new( field_series, *k, ); Ok(Value::Object(std::rc::Rc::new(view))) } else { Err(format!( "RecordSeries does not have field :{}", k.name() )) } } else { Err(format!( "Field accessor .{} expects a record or RecordSeries, got {}", k.name(), obj.type_name() )) } } else { Err(format!( "Field accessor .{} expects a record or RecordSeries, got {}", k.name(), rec )) } }; self.stack.truncate(base); return res; } Value::Object(obj) => { if let Some(closure) = obj.as_any().downcast_ref::() { self.frames.push(CallFrame { stack_base: base, closure: Some(obj.clone()), }); let unpack_res = if let Some(count) = closure.positional_count && (self.stack.len() - base) == count as usize { Ok(()) } else { let args_for_unpack = self.stack[base..].to_vec(); self.stack.truncate(base); if let BoundKind::Tuple { elements } = &closure.parameter_node.kind { let mut offset = 0; let mut res = Ok(()); for el in elements { if let Err(e) = self.unpack(el, &args_for_unpack, &mut offset) { res = Err(e); break; } } res } else { self.unpack(&closure.parameter_node, &args_for_unpack, &mut 0) } }; let res = match unpack_res { Ok(_) => self.eval_internal(obs, &closure.exec_node), Err(e) => Err(e), }; self.frames.pop(); res } else if let Some(series) = obj.as_series() { let arg_len = self.stack.len() - base; let res = if arg_len != 1 { Err(format!( "{} indexer expects exactly 1 argument (the lookback index)", obj.type_name() )) } else if let Value::Int(idx) = self.stack[base] { if idx < 0 { Err(format!( "{} lookback index cannot be negative", obj.type_name() )) } else if let Some(val) = series.get_item(idx as usize) { Ok(val) } else { Ok(Value::Void) } } else { Err(format!( "{} index must be an integer", obj.type_name() )) }; self.stack.truncate(base); return res; } else { self.stack.truncate(base); return Err(format!("Object is not callable: {}", obj.type_name())); } } _ => { self.stack.truncate(base); return Err(format!("Attempt to call non-function: {}", current_func)); } }; match result { Ok(Value::TailCallRequest(payload)) => { let (next_obj, next_args) = *payload; current_func = Value::Object(next_obj); self.stack.truncate(base); self.stack.extend(next_args); continue; } res => { self.stack.truncate(base); return res; } } } } BoundKind::Again { args } => { let base = self.stack.len(); if let Err(e) = self.eval_args_to_stack(obs, args) { self.stack.truncate(base); return Err(e); } let arg_vals = self.stack[base..].to_vec(); self.stack.truncate(base); let frame = self.frames.last().ok_or("No call frame for 'again'")?; if let Some(closure_obj) = &frame.closure { Ok(Value::TailCallRequest(Box::new(( closure_obj.clone(), arg_vals, )))) } else { Err("'again' called outside of a closure".to_string()) } } BoundKind::Tuple { elements } => { let mut vals = Vec::with_capacity(elements.len()); for e in elements { vals.push(self.eval_internal(obs, e)?); } Ok(Value::make_tuple(vals)) } BoundKind::Record { layout, values } => { let mut evaluated_values = Vec::with_capacity(values.len()); for v in values { evaluated_values.push(self.eval_internal(obs, v)?); } Ok(Value::Record( layout.clone(), std::rc::Rc::new(evaluated_values), )) } BoundKind::Expansion { bound_expanded, .. } => { let mut curr = bound_expanded; if !O::ACTIVE { while let BoundKind::Expansion { bound_expanded: next, .. } = &curr.kind { curr = next; } } self.eval_internal(obs, curr) } BoundKind::Extension(ext) => Err(format!( "Execution of extension '{}' not implemented yet", ext.display_name() )), BoundKind::Error => Err("Cannot execute a poisoned AST node".to_string()), } } fn eval_args_to_stack( &mut self, obs: &mut O, args: &ExecNode, ) -> Result<(), String> { match &args.kind { BoundKind::Tuple { elements } => { for e in elements { let mut curr = e.as_ref(); if !O::ACTIVE { while let BoundKind::Expansion { bound_expanded: next, .. } = &curr.kind { curr = next; } } match &curr.kind { BoundKind::Constant(v) if !O::ACTIVE => self.stack.push(v.clone()), _ => { let val = self.eval_internal(obs, curr)?; self.stack.push(val); } } } Ok(()) } BoundKind::Constant(v) => { if let Some(slice) = v.as_slice() { self.stack.extend_from_slice(slice); } else { self.stack.push(v.clone()); } Ok(()) } BoundKind::Expansion { bound_expanded, .. } => { let mut curr = bound_expanded; if !O::ACTIVE { while let BoundKind::Expansion { bound_expanded: next, .. } = &curr.kind { curr = next; } } let val = self.eval_internal(obs, curr)?; if let Some(slice) = val.as_slice() { self.stack.extend_from_slice(slice); } else { self.stack.push(val); } Ok(()) } _ => { let val = self.eval_internal(obs, args)?; if let Some(slice) = val.as_slice() { self.stack.extend_from_slice(slice); } else { self.stack.push(val); } Ok(()) } } } fn capture_upvalue(&mut self, addr: Address) -> Result>, String> { match addr { Address::Local(slot) => { let frame = self.frames.last().ok_or("No call frame")?; let abs_index = frame.stack_base + (slot.0 as usize); if abs_index < self.stack.len() { if let Value::Cell(cell) = &self.stack[abs_index] { Ok(cell.clone()) } else { let val = self.stack[abs_index].clone(); let cell = Rc::new(RefCell::new(val)); self.stack[abs_index] = Value::Cell(cell.clone()); Ok(cell) } } else { Err(format!("Stack underflow capture local {}", slot)) } } Address::Upvalue(idx) => { let frame = self.frames.last().ok_or("No call frame")?; if let Some(closure_obj) = &frame.closure { let closure = closure_obj.as_any().downcast_ref::().unwrap(); let u_idx = idx.0 as usize; if u_idx < closure.upvalues.len() { Ok(closure.upvalues[u_idx].clone()) } else { Err(format!("Upvalue access out of bounds capture {}", idx)) } } else { Err("Current frame has no closure".to_string()) } } Address::Global(_) => Err("Cannot capture global directly".to_string()), } } fn get_value(&self, addr: Address) -> Result { match addr { Address::Local(slot) => { let frame = self.frames.last().ok_or("No call frame")?; let abs_index = frame.stack_base + (slot.0 as usize); if abs_index < self.stack.len() { match &self.stack[abs_index] { Value::Cell(cell) => Ok(cell.borrow().clone()), val => Ok(val.clone()), } } else { Err(format!("Stack underflow access local {}", slot)) } } Address::Global(idx) => { let g_idx = idx.0 as usize; let globals = self.globals.borrow(); if g_idx < globals.len() { Ok(globals[g_idx].clone()) } else { Err(format!("Global access out of bounds {}", idx)) } } Address::Upvalue(idx) => { let frame = self.frames.last().ok_or("No call frame")?; if let Some(closure_obj) = &frame.closure { let closure = closure_obj.as_any().downcast_ref::().unwrap(); let u_idx = idx.0 as usize; if u_idx < closure.upvalues.len() { Ok(closure.upvalues[u_idx].borrow().clone()) } else { Err(format!("Upvalue access out of bounds {}", idx)) } } else { Err("Current frame has no closure (cannot access upvalues)".to_string()) } } } } fn set_value(&mut self, addr: Address, value: Value) -> Result<(), String> { match addr { Address::Local(slot) => { let frame = self.frames.last().ok_or("No call frame")?; let abs_index = frame.stack_base + (slot.0 as usize); if abs_index < self.stack.len() { if let Value::Cell(cell) = &self.stack[abs_index] { *cell.borrow_mut() = value; } else { self.stack[abs_index] = value; } } else if abs_index == self.stack.len() { self.stack.push(value); } else { return Err(format!("Stack gap write local {}", slot)); } Ok(()) } Address::Global(idx) => { let g_idx = idx.0 as usize; let mut globals = self.globals.borrow_mut(); if g_idx >= globals.len() { globals.resize(g_idx + 1, Value::Void); } globals[g_idx] = value; Ok(()) } Address::Upvalue(idx) => { let frame = self.frames.last().ok_or("No call frame")?; if let Some(closure_obj) = &frame.closure { let closure = closure_obj.as_any().downcast_ref::().unwrap(); let u_idx = idx.0 as usize; if u_idx < closure.upvalues.len() { *closure.upvalues[u_idx].borrow_mut() = value; Ok(()) } else { Err(format!("Upvalue assignment out of bounds {}", idx)) } } else { Err("Current frame has no closure".to_string()) } } } } fn unpack( &mut self, pattern: &Node, T>, values: &[Value], offset: &mut usize, ) -> Result<(), String> { match &pattern.kind { BoundKind::Define { addr, .. } => { let val = values.get(*offset).cloned().unwrap_or(Value::Void); *offset += 1; self.set_value(*addr, val) } BoundKind::Set { addr, .. } => { let val = values.get(*offset).cloned().unwrap_or(Value::Void); *offset += 1; self.set_value(*addr, val) } BoundKind::Tuple { elements } => { if let Some(sub_values) = values.get(*offset).and_then(|v| v.as_slice()) { *offset += 1; let mut sub_offset = 0; for el in elements { self.unpack(el, sub_values, &mut sub_offset)?; } return Ok(()); } for el in elements { self.unpack(el, values, offset)?; } Ok(()) } _ => Err("Invalid node in parameter pattern".to_string()), } } }