From 2eb7a2e1368eb4cbbf7f7eab6b79a5e32f46263b Mon Sep 17 00:00:00 2001 From: Michael Schimmel Date: Mon, 2 Mar 2026 23:13:36 +0100 Subject: [PATCH] Formatting --- src/ast/compiler/analyzer.rs | 14 +- src/ast/compiler/binder.rs | 1446 +++++++++++++------------- src/ast/compiler/bound_nodes.rs | 20 +- src/ast/compiler/captures.rs | 6 +- src/ast/compiler/dumper.rs | 9 +- src/ast/compiler/optimizer/engine.rs | 27 +- src/ast/compiler/optimizer/folder.rs | 2 +- src/ast/compiler/optimizer/utils.rs | 5 +- src/ast/compiler/specializer.rs | 5 +- src/ast/compiler/tco.rs | 6 +- src/ast/compiler/type_checker.rs | 87 +- src/ast/environment.rs | 72 +- src/ast/parser.rs | 21 +- src/ast/rtl/math.rs | 349 ++++--- src/ast/rtl/mod.rs | 2 +- src/ast/rtl/series.rs | 1088 +++++++++---------- src/ast/rtl/streams.rs | 1234 +++++++++++----------- src/ast/rtl/type_registry.rs | 7 +- src/ast/types.rs | 47 +- src/ast/vm.rs | 126 ++- src/bin/ast.rs | 7 +- src/integration_test.rs | 1445 +++++++++++++------------ 22 files changed, 3212 insertions(+), 2813 deletions(-) diff --git a/src/ast/compiler/analyzer.rs b/src/ast/compiler/analyzer.rs index 648c17d..2903209 100644 --- a/src/ast/compiler/analyzer.rs +++ b/src/ast/compiler/analyzer.rs @@ -237,7 +237,11 @@ impl<'a> Analyzer<'a> { Purity::Impure, ) } - BoundKind::Pipe { inputs, lambda, out_type } => { + BoundKind::Pipe { + inputs, + lambda, + out_type, + } => { let mut analyzed_inputs = Vec::with_capacity(inputs.len()); for input in inputs { analyzed_inputs.push(self.visit(Rc::new(input.clone()))); @@ -288,7 +292,13 @@ impl<'a> Analyzer<'a> { p = p.min(vm.ty.purity); new_values.push(vm); } - (BoundKind::Record { layout: layout.clone(), values: new_values }, p) + ( + BoundKind::Record { + layout: layout.clone(), + values: new_values, + }, + p, + ) } BoundKind::Expansion { diff --git a/src/ast/compiler/binder.rs b/src/ast/compiler/binder.rs index 9f66ddf..857aa9f 100644 --- a/src/ast/compiler/binder.rs +++ b/src/ast/compiler/binder.rs @@ -1,708 +1,738 @@ -use crate::ast::compiler::bound_nodes::{ - Address, BoundKind, BoundNode, DeclarationKind, GlobalIdx, LocalSlot, UpvalueIdx, -}; -use crate::ast::diagnostics::Diagnostics; -use crate::ast::nodes::{Node, Symbol, UntypedKind}; -use crate::ast::types::{Identity, StaticType}; -use std::cell::RefCell; -use std::collections::HashMap; -use std::rc::Rc; - -#[derive(Debug, Clone)] -struct LocalInfo { - slot: LocalSlot, - identity: Identity, - // Note: Binder doesn't strictly need the type anymore, - // but it might be useful for built-ins during resolution. - // For now we keep it as Any or Unknown. - _ty: StaticType, -} - -#[derive(Debug, Clone)] -struct CompilerScope { - locals: HashMap, - slot_count: u32, -} - -impl CompilerScope { - fn new() -> Self { - Self { - locals: HashMap::new(), - slot_count: 0, - } - } - - fn define(&mut self, sym: &Symbol, identity: Identity) -> Result { - if self.locals.contains_key(sym) { - return Err(format!( - "Variable '{}' is already defined in this scope level.", - sym.name - )); - } - let slot = LocalSlot(self.slot_count); - self.locals.insert( - sym.clone(), - LocalInfo { - slot, - identity, - _ty: StaticType::Any, - }, - ); - self.slot_count += 1; - Ok(slot) - } - - fn resolve(&self, sym: &Symbol) -> Option { - self.locals.get(sym).cloned() - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ScopeKind { - Root, - Local, -} - -struct FunctionCompiler { - identity: Identity, - scope: CompilerScope, - upvalues: Vec
, - kind: ScopeKind, -} - -impl FunctionCompiler { - fn new(kind: ScopeKind, identity: Identity) -> Self { - Self { - identity, - scope: CompilerScope::new(), - upvalues: Vec::new(), - kind, - } - } - - fn define_variable( - &mut self, - name: &Symbol, - identity: Identity, - globals: &Rc>>, - ) -> Result { - match self.kind { - ScopeKind::Root => { - let mut globals_map = globals.borrow_mut(); - if globals_map.contains_key(name) { - return Err(format!( - "Global variable '{}' is already defined.", - name.name - )); - } - let idx = globals_map.len() as u32; - globals_map.insert(name.clone(), (GlobalIdx(idx), identity)); - Ok(Address::Global(GlobalIdx(idx))) - } - ScopeKind::Local => { - let slot = self.scope.define(name, identity)?; - Ok(Address::Local(slot)) - } - } - } - - fn add_upvalue(&mut self, addr: Address) -> UpvalueIdx { - if let Some(idx) = self.upvalues.iter().position(|&a| a == addr) { - return UpvalueIdx(idx as u32); - } - let idx = UpvalueIdx(self.upvalues.len() as u32); - self.upvalues.push(addr); - idx - } -} - -pub struct Binder { - functions: Vec, - // Globals mapping: Symbol -> (Index, DefinitionIdentity) - globals: Rc>>, - // Map of Declaration Identity -> List of Lambda Identities that capture it - capture_map: HashMap>, -} - -impl Binder { - pub fn new(globals: Rc>>) -> Self { - let mut binder = Self { - functions: Vec::new(), - globals, - capture_map: HashMap::new(), - }; - binder.functions.push(FunctionCompiler::new( - ScopeKind::Root, - crate::ast::types::NodeIdentity::new(crate::ast::types::SourceLocation { - line: 0, - col: 0, - }), - )); - binder - } - - pub fn bind_root( - globals: Rc>>, - node: &Node, - diagnostics: &mut Diagnostics, - ) -> Result<(BoundNode, HashMap>), String> { - let mut binder = Self::new(globals); - let bound = binder.bind(node, diagnostics); - - // Convert HashSet to sorted Vec - let final_captures = binder - .capture_map - .into_iter() - .map(|(k, v)| (k, v.into_iter().collect())) - .collect(); - - Ok((bound, final_captures)) - } - - fn declare_variable( - &mut self, - name: &Symbol, - identity: Identity, - _kind: crate::ast::compiler::bound_nodes::DeclarationKind, - diag: &mut Diagnostics, - ) -> Option
{ - let current_fn = self.functions.last_mut().unwrap(); - match current_fn.define_variable(name, identity, &self.globals) { - Ok(addr) => Some(addr), - Err(e) => { - diag.push_error(e, None); - None - } - } - } - - pub fn bind(&mut self, node: &Node, diag: &mut Diagnostics) -> BoundNode { - match &node.kind { - UntypedKind::Nop => self.make_node(node.identity.clone(), BoundKind::Nop), - UntypedKind::Constant(v) => { - self.make_node(node.identity.clone(), BoundKind::Constant(v.clone())) - } - - UntypedKind::Identifier(sym) => { - if let Some(addr) = self.resolve_variable(sym, diag, &node.identity) { - self.make_node( - node.identity.clone(), - BoundKind::Get { - addr, - name: sym.clone(), - }, - ) - } else { - self.make_node(node.identity.clone(), BoundKind::Error) - } - } - - UntypedKind::Parameter(_) => { - diag.push_error("Unexpected 'Parameter' node in general binder context. This should be handled via 'bind_pattern'.", Some(node.identity.clone())); - self.make_node(node.identity.clone(), BoundKind::Error) - } - - UntypedKind::FieldAccessor(k) => { - self.make_node(node.identity.clone(), BoundKind::FieldAccessor(*k)) - } - - UntypedKind::If { - cond, - then_br, - else_br, - } => { - let cond = self.bind(cond, diag); - let then_br = self.bind(then_br, diag); - let mut else_br_bound = None; - if let Some(e) = else_br { - else_br_bound = Some(Box::new(self.bind(e, diag))); - } - - self.make_node( - node.identity.clone(), - BoundKind::If { - cond: Box::new(cond), - then_br: Box::new(then_br), - else_br: else_br_bound, - }, - ) - } - - UntypedKind::Def { target, value } => { - // Special case: Single identifier (to support recursion) - if let UntypedKind::Parameter(ref name) = target.kind { - let addr_opt = self.declare_variable( - name, - node.identity.clone(), // Identity of the Def node - crate::ast::compiler::bound_nodes::DeclarationKind::Variable, - diag, - ); - let val_node = self.bind(value, diag); - - if let Some(addr) = addr_opt { - self.make_node( - node.identity.clone(), - BoundKind::Define { - name: name.clone(), - addr, - kind: crate::ast::compiler::bound_nodes::DeclarationKind::Variable, - value: Box::new(val_node), - captured_by: Vec::new(), // Will be filled by post-pass - }, - ) - } else { - self.make_node(node.identity.clone(), BoundKind::Error) - } - } else { - // Complex Destructuring Pattern - // NOTE: Destructuring definitions are NOT recursive by default - // (the variables are only available AFTER the definition) - let val_node = self.bind(value, diag); - let target_node = self.bind_pattern(target, DeclarationKind::Variable, diag); - - self.make_node( - node.identity.clone(), - BoundKind::Destructure { - pattern: Box::new(target_node), - value: Box::new(val_node), - }, - ) - } - } - - UntypedKind::Assign { target, value } => { - let val_node = self.bind(value, diag); - - if let UntypedKind::Identifier(sym) = &target.kind { - if let Some(addr) = self.resolve_variable(sym, diag, &target.identity) { - self.make_node( - node.identity.clone(), - BoundKind::Set { - addr, - value: Box::new(val_node), - }, - ) - } else { - self.make_node(node.identity.clone(), BoundKind::Error) - } - } else { - let target_node = self.bind_assign_pattern(target, diag); - self.make_node( - node.identity.clone(), - BoundKind::Destructure { - pattern: Box::new(target_node), - value: Box::new(val_node), - }, - ) - } - } - - UntypedKind::Pipe { inputs, lambda } => { - let mut bound_inputs = Vec::with_capacity(inputs.len()); - for input in inputs { - bound_inputs.push(self.bind(input, diag)); - } - let bound_lambda = Box::new(self.bind(lambda.as_ref(), diag)); - self.make_node( - node.identity.clone(), - BoundKind::Pipe { - inputs: bound_inputs, - lambda: bound_lambda, - out_type: crate::ast::types::StaticType::Any, - }, - ) - } - - UntypedKind::Lambda { params, body } => { - let identity = node.identity.clone(); - self.functions.push(FunctionCompiler::new( - ScopeKind::Local, - identity.clone(), - )); - - // 1. Bind the parameter pattern/tuple - let params_bound = self.bind_pattern(params, DeclarationKind::Parameter, diag); - - // 2. Bind the body - let body_bound = self.bind(body, diag); - - let compiled_fn = self.functions.pop().unwrap(); - - // 3. Static optimization: count total parameters needed in flat argument list - fn count_params(node: &BoundNode) -> Option { - match &node.kind { - BoundKind::Define { - kind: DeclarationKind::Parameter, - .. - } => Some(1), - BoundKind::Tuple { elements } => { - let mut total = 0; - for e in elements { - total += count_params(e)?; - } - Some(total) - } - BoundKind::Nop => Some(0), - _ => None, - } - } - - let positional_count = count_params(¶ms_bound); - - self.make_node( - identity, - BoundKind::Lambda { - params: Rc::new(params_bound), - upvalues: compiled_fn.upvalues, - body: Rc::new(body_bound), - positional_count, - }, - ) - } - - UntypedKind::Call { callee, args } => { - let callee = self.bind(callee, diag); - let args = self.bind(args, diag); - - self.make_node( - node.identity.clone(), - BoundKind::Call { - callee: Box::new(callee), - args: Box::new(args), - }, - ) - } - - UntypedKind::Again { args } => { - if self.functions.len() <= 1 { - diag.push_error("'again' is only allowed inside a function or lambda.", Some(node.identity.clone())); - return self.make_node(node.identity.clone(), BoundKind::Error); - } - let args = self.bind(args, diag); - self.make_node( - node.identity.clone(), - BoundKind::Again { - args: Box::new(args), - }, - ) - } - - UntypedKind::Block { exprs } => { - let mut bound_exprs = Vec::new(); - for expr in exprs { - bound_exprs.push(self.bind(expr, diag)); - } - self.make_node( - node.identity.clone(), - BoundKind::Block { exprs: bound_exprs }, - ) - } - - UntypedKind::Tuple { elements } => { - let mut bound_elems = Vec::new(); - for e in elements { - bound_elems.push(self.bind(e, diag)); - } - self.make_node( - node.identity.clone(), - BoundKind::Tuple { - elements: bound_elems, - }, - ) - } - - UntypedKind::Record { fields } => { - let mut bound_values = Vec::new(); - let mut layout_fields = Vec::new(); - - for (k, v) in fields { - let key_node = self.bind(k, diag); - let val_node = self.bind(v, diag); - - if let BoundKind::Constant(crate::ast::types::Value::Keyword(kw)) = key_node.kind { - layout_fields.push((kw, crate::ast::types::StaticType::Any)); - } else { - diag.push_error(format!("Record keys must be keywords, found at {:?}", key_node.identity.location), Some(key_node.identity.clone())); - } - bound_values.push(val_node); - } - - let layout = crate::ast::types::RecordLayout::get_or_create(layout_fields); - - self.make_node( - node.identity.clone(), - BoundKind::Record { - layout, - values: bound_values, - }, - ) - } - - UntypedKind::Expansion { call, expanded } => { - let bound_expanded = self.bind(expanded, diag); - self.make_node( - node.identity.clone(), - BoundKind::Expansion { - original_call: Rc::from(call.as_ref().clone()), - bound_expanded: Box::new(bound_expanded), - }, - ) - } - - UntypedKind::Template(_) - | UntypedKind::Placeholder(_) - | UntypedKind::Splice(_) - | UntypedKind::MacroDecl { .. } => { - diag.push_error(format!("Macro construct {:?} found in Binder. Macros must be expanded before binding.", node.kind), Some(node.identity.clone())); - self.make_node(node.identity.clone(), BoundKind::Error) - } - - UntypedKind::Extension(_) => { - diag.push_error("Custom extensions not supported in Binder yet", Some(node.identity.clone())); - self.make_node(node.identity.clone(), BoundKind::Error) - } - UntypedKind::Error => crate::ast::compiler::bound_nodes::BoundNode { - identity: node.identity.clone(), - kind: crate::ast::compiler::bound_nodes::BoundKind::Error, - ty: (), - }, - } - } - - fn resolve_variable(&mut self, sym: &Symbol, diag: &mut Diagnostics, identity: &Identity) -> Option
{ - let current_fn_idx = self.functions.len() - 1; - - // 1. Try local in current function - if let Some(info) = self.functions[current_fn_idx].scope.resolve(sym) { - return Some(Address::Local(info.slot)); - } - - // 2. Try enclosing scopes (capture chain) - for i in (0..current_fn_idx).rev() { - if let Some(info) = self.functions[i].scope.resolve(sym) { - let mut addr = Address::Local(info.slot); - - // Record the capture for each lambda level in between - for k in (i + 1)..=current_fn_idx { - let lambda_id = self.functions[k].identity.clone(); - self.capture_map - .entry(info.identity.clone()) - .or_default() - .insert(lambda_id); - addr = Address::Upvalue(self.functions[k].add_upvalue(addr)); - } - return Some(addr); - } - } - - // 3. Try Global - let globals = self.globals.borrow(); - if let Some((idx, _)) = globals.get(sym) { - return Some(Address::Global(*idx)); - } - - // 4. Global Fallback - if sym.context.is_some() { - let fallback_sym = Symbol { - name: sym.name.clone(), - context: None, - }; - if let Some((idx, _)) = globals.get(&fallback_sym) { - return Some(Address::Global(*idx)); - } - } - - diag.push_error(format!("Undefined variable '{}'", sym.name), Some(identity.clone())); - None - } - - fn bind_pattern( - &mut self, - node: &Node, - kind: DeclarationKind, - diag: &mut Diagnostics, - ) -> BoundNode { - match &node.kind { - UntypedKind::Parameter(sym) => { - if let Some(addr) = self.declare_variable(sym, node.identity.clone(), kind, diag) { - self.make_node( - node.identity.clone(), - BoundKind::Define { - name: sym.clone(), - addr, - kind, - value: Box::new(self.make_node(node.identity.clone(), BoundKind::Nop)), - captured_by: Vec::new(), // Filled by post-pass - }, - ) - } else { - self.make_node(node.identity.clone(), BoundKind::Error) - } - } - UntypedKind::Tuple { elements } => { - let mut bound_elems = Vec::new(); - for e in elements { - bound_elems.push(self.bind_pattern(e, kind, diag)); - } - self.make_node( - node.identity.clone(), - BoundKind::Tuple { - elements: bound_elems, - }, - ) - } - _ => { - diag.push_error(format!("Invalid node in pattern: {:?}", node.kind), Some(node.identity.clone())); - self.make_node(node.identity.clone(), BoundKind::Error) - } - } - } - - fn bind_assign_pattern(&mut self, node: &Node, diag: &mut Diagnostics) -> BoundNode { - match &node.kind { - UntypedKind::Identifier(sym) => { - if let Some(addr) = self.resolve_variable(sym, diag, &node.identity) { - self.make_node( - node.identity.clone(), - BoundKind::Set { - addr, - value: Box::new(self.make_node(node.identity.clone(), BoundKind::Nop)), - }, - ) - } else { - self.make_node(node.identity.clone(), BoundKind::Error) - } - } - UntypedKind::Tuple { elements } => { - let mut bound_elems = Vec::new(); - for e in elements { - bound_elems.push(self.bind_assign_pattern(e, diag)); - } - self.make_node( - node.identity.clone(), - BoundKind::Tuple { - elements: bound_elems, - }, - ) - } - _ => { - diag.push_error(format!("Invalid node in assignment pattern: {:?}", node.kind), Some(node.identity.clone())); - self.make_node(node.identity.clone(), BoundKind::Error) - } - } - } - - fn make_node(&self, identity: Identity, kind: BoundKind<()>) -> BoundNode { - Node { - identity, - kind, - ty: (), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ast::parser::Parser; - use crate::ast::diagnostics::Diagnostics; - - #[test] - fn test_upvalue_capture_sets_is_boxed() { - // Wrap in a lambda to ensure 'x' is a local variable, not a global - let source = "(fn [] (do (def x 10) (def f (fn [] x)) x))"; - let mut parser = Parser::new(source); - let untyped = parser.parse_expression(); - - let globals = Rc::new(RefCell::new(HashMap::new())); - let mut diagnostics = Diagnostics::new(); - let (bound, captures) = Binder::bind_root(globals, &untyped, &mut diagnostics).unwrap(); - - // Structure: Lambda -> Block -> [ Define(x), Define(f), Get(x) ] - if let BoundKind::Lambda { body, .. } = &bound.kind { - if let BoundKind::Block { exprs } = &body.kind { - let x_decl = &exprs[0]; - if let BoundKind::Define { addr, .. } = &x_decl.kind { - assert!(matches!(addr, Address::Local(_))); - assert!( - captures.contains_key(&x_decl.identity), - "Variable 'x' should have capturers because it is used in lambda 'f'" - ); - } else { - panic!( - "First expression in block should be Define, got {:?}", - x_decl.kind - ); - } - } else { - panic!("Lambda body should be a Block, got {:?}", body.kind); - } - } else { - panic!("Root should be a Lambda, got {:?}", bound.kind); - } - } - - #[test] - fn test_no_capture_not_boxed() { - let source = "(fn [] (do (def x 10) x))"; - let mut parser = Parser::new(source); - let untyped = parser.parse_expression(); - - let globals = Rc::new(RefCell::new(HashMap::new())); - let mut diagnostics = Diagnostics::new(); - let (bound, captures) = Binder::bind_root(globals, &untyped, &mut diagnostics).unwrap(); - - if let BoundKind::Lambda { body, .. } = &bound.kind { - if let BoundKind::Block { exprs } = &body.kind { - let x_decl = &exprs[0]; - if let BoundKind::Define { addr, .. } = &x_decl.kind { - assert!(matches!(addr, Address::Local(_))); - assert!( - !captures.contains_key(&x_decl.identity), - "Variable 'x' should NOT have any capturers" - ); - } else { - panic!("First expression should be Define"); - } - } else { - panic!("Lambda body should be a Block"); - } - } else { - panic!("Root should be a Lambda"); - } - } - - #[test] - fn test_redefinition_error() { - let source = "(do (def x 1) (def x 2))"; - let mut parser = Parser::new(source); - let untyped = parser.parse_expression(); - - let globals = Rc::new(RefCell::new(HashMap::new())); - let mut diagnostics = Diagnostics::new(); - let _ = Binder::bind_root(globals, &untyped, &mut diagnostics); - - assert!(diagnostics.has_errors()); - assert!(diagnostics.items[0].message.contains("already defined")); - } - - #[test] - fn test_repro_global_redefinition() { - let globals = Rc::new(RefCell::new(HashMap::new())); - - // First run: defines 'x' - let source1 = "(def x 1)"; - let untyped1 = Parser::new(source1).parse_expression(); - let mut diagnostics = Diagnostics::new(); - assert!(Binder::bind_root(globals.clone(), &untyped1, &mut diagnostics).is_ok()); - - // Second run: attempts to redefine 'x' in the same global environment - let source2 = "(def x 2)"; - let untyped2 = Parser::new(source2).parse_expression(); - let mut diagnostics2 = Diagnostics::new(); - let _ = Binder::bind_root(globals.clone(), &untyped2, &mut diagnostics2); - - assert!(diagnostics2.has_errors()); - assert!(diagnostics2.items[0].message.contains("already defined")); - } -} +use crate::ast::compiler::bound_nodes::{ + Address, BoundKind, BoundNode, DeclarationKind, GlobalIdx, LocalSlot, UpvalueIdx, +}; +use crate::ast::diagnostics::Diagnostics; +use crate::ast::nodes::{Node, Symbol, UntypedKind}; +use crate::ast::types::{Identity, StaticType}; +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; + +#[derive(Debug, Clone)] +struct LocalInfo { + slot: LocalSlot, + identity: Identity, + // Note: Binder doesn't strictly need the type anymore, + // but it might be useful for built-ins during resolution. + // For now we keep it as Any or Unknown. + _ty: StaticType, +} + +#[derive(Debug, Clone)] +struct CompilerScope { + locals: HashMap, + slot_count: u32, +} + +impl CompilerScope { + fn new() -> Self { + Self { + locals: HashMap::new(), + slot_count: 0, + } + } + + fn define(&mut self, sym: &Symbol, identity: Identity) -> Result { + if self.locals.contains_key(sym) { + return Err(format!( + "Variable '{}' is already defined in this scope level.", + sym.name + )); + } + let slot = LocalSlot(self.slot_count); + self.locals.insert( + sym.clone(), + LocalInfo { + slot, + identity, + _ty: StaticType::Any, + }, + ); + self.slot_count += 1; + Ok(slot) + } + + fn resolve(&self, sym: &Symbol) -> Option { + self.locals.get(sym).cloned() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ScopeKind { + Root, + Local, +} + +struct FunctionCompiler { + identity: Identity, + scope: CompilerScope, + upvalues: Vec
, + kind: ScopeKind, +} + +impl FunctionCompiler { + fn new(kind: ScopeKind, identity: Identity) -> Self { + Self { + identity, + scope: CompilerScope::new(), + upvalues: Vec::new(), + kind, + } + } + + fn define_variable( + &mut self, + name: &Symbol, + identity: Identity, + globals: &Rc>>, + ) -> Result { + match self.kind { + ScopeKind::Root => { + let mut globals_map = globals.borrow_mut(); + if globals_map.contains_key(name) { + return Err(format!( + "Global variable '{}' is already defined.", + name.name + )); + } + let idx = globals_map.len() as u32; + globals_map.insert(name.clone(), (GlobalIdx(idx), identity)); + Ok(Address::Global(GlobalIdx(idx))) + } + ScopeKind::Local => { + let slot = self.scope.define(name, identity)?; + Ok(Address::Local(slot)) + } + } + } + + fn add_upvalue(&mut self, addr: Address) -> UpvalueIdx { + if let Some(idx) = self.upvalues.iter().position(|&a| a == addr) { + return UpvalueIdx(idx as u32); + } + let idx = UpvalueIdx(self.upvalues.len() as u32); + self.upvalues.push(addr); + idx + } +} + +pub struct Binder { + functions: Vec, + // Globals mapping: Symbol -> (Index, DefinitionIdentity) + globals: Rc>>, + // Map of Declaration Identity -> List of Lambda Identities that capture it + capture_map: HashMap>, +} + +impl Binder { + pub fn new(globals: Rc>>) -> Self { + let mut binder = Self { + functions: Vec::new(), + globals, + capture_map: HashMap::new(), + }; + binder.functions.push(FunctionCompiler::new( + ScopeKind::Root, + crate::ast::types::NodeIdentity::new(crate::ast::types::SourceLocation { + line: 0, + col: 0, + }), + )); + binder + } + + pub fn bind_root( + globals: Rc>>, + node: &Node, + diagnostics: &mut Diagnostics, + ) -> Result<(BoundNode, HashMap>), String> { + let mut binder = Self::new(globals); + let bound = binder.bind(node, diagnostics); + + // Convert HashSet to sorted Vec + let final_captures = binder + .capture_map + .into_iter() + .map(|(k, v)| (k, v.into_iter().collect())) + .collect(); + + Ok((bound, final_captures)) + } + + fn declare_variable( + &mut self, + name: &Symbol, + identity: Identity, + _kind: crate::ast::compiler::bound_nodes::DeclarationKind, + diag: &mut Diagnostics, + ) -> Option
{ + let current_fn = self.functions.last_mut().unwrap(); + match current_fn.define_variable(name, identity, &self.globals) { + Ok(addr) => Some(addr), + Err(e) => { + diag.push_error(e, None); + None + } + } + } + + pub fn bind(&mut self, node: &Node, diag: &mut Diagnostics) -> BoundNode { + match &node.kind { + UntypedKind::Nop => self.make_node(node.identity.clone(), BoundKind::Nop), + UntypedKind::Constant(v) => { + self.make_node(node.identity.clone(), BoundKind::Constant(v.clone())) + } + + UntypedKind::Identifier(sym) => { + if let Some(addr) = self.resolve_variable(sym, diag, &node.identity) { + self.make_node( + node.identity.clone(), + BoundKind::Get { + addr, + name: sym.clone(), + }, + ) + } else { + self.make_node(node.identity.clone(), BoundKind::Error) + } + } + + UntypedKind::Parameter(_) => { + diag.push_error("Unexpected 'Parameter' node in general binder context. This should be handled via 'bind_pattern'.", Some(node.identity.clone())); + self.make_node(node.identity.clone(), BoundKind::Error) + } + + UntypedKind::FieldAccessor(k) => { + self.make_node(node.identity.clone(), BoundKind::FieldAccessor(*k)) + } + + UntypedKind::If { + cond, + then_br, + else_br, + } => { + let cond = self.bind(cond, diag); + let then_br = self.bind(then_br, diag); + let mut else_br_bound = None; + if let Some(e) = else_br { + else_br_bound = Some(Box::new(self.bind(e, diag))); + } + + self.make_node( + node.identity.clone(), + BoundKind::If { + cond: Box::new(cond), + then_br: Box::new(then_br), + else_br: else_br_bound, + }, + ) + } + + UntypedKind::Def { target, value } => { + // Special case: Single identifier (to support recursion) + if let UntypedKind::Parameter(ref name) = target.kind { + let addr_opt = self.declare_variable( + name, + node.identity.clone(), // Identity of the Def node + crate::ast::compiler::bound_nodes::DeclarationKind::Variable, + diag, + ); + let val_node = self.bind(value, diag); + + if let Some(addr) = addr_opt { + self.make_node( + node.identity.clone(), + BoundKind::Define { + name: name.clone(), + addr, + kind: crate::ast::compiler::bound_nodes::DeclarationKind::Variable, + value: Box::new(val_node), + captured_by: Vec::new(), // Will be filled by post-pass + }, + ) + } else { + self.make_node(node.identity.clone(), BoundKind::Error) + } + } else { + // Complex Destructuring Pattern + // NOTE: Destructuring definitions are NOT recursive by default + // (the variables are only available AFTER the definition) + let val_node = self.bind(value, diag); + let target_node = self.bind_pattern(target, DeclarationKind::Variable, diag); + + self.make_node( + node.identity.clone(), + BoundKind::Destructure { + pattern: Box::new(target_node), + value: Box::new(val_node), + }, + ) + } + } + + UntypedKind::Assign { target, value } => { + let val_node = self.bind(value, diag); + + if let UntypedKind::Identifier(sym) = &target.kind { + if let Some(addr) = self.resolve_variable(sym, diag, &target.identity) { + self.make_node( + node.identity.clone(), + BoundKind::Set { + addr, + value: Box::new(val_node), + }, + ) + } else { + self.make_node(node.identity.clone(), BoundKind::Error) + } + } else { + let target_node = self.bind_assign_pattern(target, diag); + self.make_node( + node.identity.clone(), + BoundKind::Destructure { + pattern: Box::new(target_node), + value: Box::new(val_node), + }, + ) + } + } + + UntypedKind::Pipe { inputs, lambda } => { + let mut bound_inputs = Vec::with_capacity(inputs.len()); + for input in inputs { + bound_inputs.push(self.bind(input, diag)); + } + let bound_lambda = Box::new(self.bind(lambda.as_ref(), diag)); + self.make_node( + node.identity.clone(), + BoundKind::Pipe { + inputs: bound_inputs, + lambda: bound_lambda, + out_type: crate::ast::types::StaticType::Any, + }, + ) + } + + UntypedKind::Lambda { params, body } => { + let identity = node.identity.clone(); + self.functions + .push(FunctionCompiler::new(ScopeKind::Local, identity.clone())); + + // 1. Bind the parameter pattern/tuple + let params_bound = self.bind_pattern(params, DeclarationKind::Parameter, diag); + + // 2. Bind the body + let body_bound = self.bind(body, diag); + + let compiled_fn = self.functions.pop().unwrap(); + + // 3. Static optimization: count total parameters needed in flat argument list + fn count_params(node: &BoundNode) -> Option { + match &node.kind { + BoundKind::Define { + kind: DeclarationKind::Parameter, + .. + } => Some(1), + BoundKind::Tuple { elements } => { + let mut total = 0; + for e in elements { + total += count_params(e)?; + } + Some(total) + } + BoundKind::Nop => Some(0), + _ => None, + } + } + + let positional_count = count_params(¶ms_bound); + + self.make_node( + identity, + BoundKind::Lambda { + params: Rc::new(params_bound), + upvalues: compiled_fn.upvalues, + body: Rc::new(body_bound), + positional_count, + }, + ) + } + + UntypedKind::Call { callee, args } => { + let callee = self.bind(callee, diag); + let args = self.bind(args, diag); + + self.make_node( + node.identity.clone(), + BoundKind::Call { + callee: Box::new(callee), + args: Box::new(args), + }, + ) + } + + UntypedKind::Again { args } => { + if self.functions.len() <= 1 { + diag.push_error( + "'again' is only allowed inside a function or lambda.", + Some(node.identity.clone()), + ); + return self.make_node(node.identity.clone(), BoundKind::Error); + } + let args = self.bind(args, diag); + self.make_node( + node.identity.clone(), + BoundKind::Again { + args: Box::new(args), + }, + ) + } + + UntypedKind::Block { exprs } => { + let mut bound_exprs = Vec::new(); + for expr in exprs { + bound_exprs.push(self.bind(expr, diag)); + } + self.make_node( + node.identity.clone(), + BoundKind::Block { exprs: bound_exprs }, + ) + } + + UntypedKind::Tuple { elements } => { + let mut bound_elems = Vec::new(); + for e in elements { + bound_elems.push(self.bind(e, diag)); + } + self.make_node( + node.identity.clone(), + BoundKind::Tuple { + elements: bound_elems, + }, + ) + } + + UntypedKind::Record { fields } => { + let mut bound_values = Vec::new(); + let mut layout_fields = Vec::new(); + + for (k, v) in fields { + let key_node = self.bind(k, diag); + let val_node = self.bind(v, diag); + + if let BoundKind::Constant(crate::ast::types::Value::Keyword(kw)) = + key_node.kind + { + layout_fields.push((kw, crate::ast::types::StaticType::Any)); + } else { + diag.push_error( + format!( + "Record keys must be keywords, found at {:?}", + key_node.identity.location + ), + Some(key_node.identity.clone()), + ); + } + bound_values.push(val_node); + } + + let layout = crate::ast::types::RecordLayout::get_or_create(layout_fields); + + self.make_node( + node.identity.clone(), + BoundKind::Record { + layout, + values: bound_values, + }, + ) + } + + UntypedKind::Expansion { call, expanded } => { + let bound_expanded = self.bind(expanded, diag); + self.make_node( + node.identity.clone(), + BoundKind::Expansion { + original_call: Rc::from(call.as_ref().clone()), + bound_expanded: Box::new(bound_expanded), + }, + ) + } + + UntypedKind::Template(_) + | UntypedKind::Placeholder(_) + | UntypedKind::Splice(_) + | UntypedKind::MacroDecl { .. } => { + diag.push_error(format!("Macro construct {:?} found in Binder. Macros must be expanded before binding.", node.kind), Some(node.identity.clone())); + self.make_node(node.identity.clone(), BoundKind::Error) + } + + UntypedKind::Extension(_) => { + diag.push_error( + "Custom extensions not supported in Binder yet", + Some(node.identity.clone()), + ); + self.make_node(node.identity.clone(), BoundKind::Error) + } + UntypedKind::Error => crate::ast::compiler::bound_nodes::BoundNode { + identity: node.identity.clone(), + kind: crate::ast::compiler::bound_nodes::BoundKind::Error, + ty: (), + }, + } + } + + fn resolve_variable( + &mut self, + sym: &Symbol, + diag: &mut Diagnostics, + identity: &Identity, + ) -> Option
{ + let current_fn_idx = self.functions.len() - 1; + + // 1. Try local in current function + if let Some(info) = self.functions[current_fn_idx].scope.resolve(sym) { + return Some(Address::Local(info.slot)); + } + + // 2. Try enclosing scopes (capture chain) + for i in (0..current_fn_idx).rev() { + if let Some(info) = self.functions[i].scope.resolve(sym) { + let mut addr = Address::Local(info.slot); + + // Record the capture for each lambda level in between + for k in (i + 1)..=current_fn_idx { + let lambda_id = self.functions[k].identity.clone(); + self.capture_map + .entry(info.identity.clone()) + .or_default() + .insert(lambda_id); + addr = Address::Upvalue(self.functions[k].add_upvalue(addr)); + } + return Some(addr); + } + } + + // 3. Try Global + let globals = self.globals.borrow(); + if let Some((idx, _)) = globals.get(sym) { + return Some(Address::Global(*idx)); + } + + // 4. Global Fallback + if sym.context.is_some() { + let fallback_sym = Symbol { + name: sym.name.clone(), + context: None, + }; + if let Some((idx, _)) = globals.get(&fallback_sym) { + return Some(Address::Global(*idx)); + } + } + + diag.push_error( + format!("Undefined variable '{}'", sym.name), + Some(identity.clone()), + ); + None + } + + fn bind_pattern( + &mut self, + node: &Node, + kind: DeclarationKind, + diag: &mut Diagnostics, + ) -> BoundNode { + match &node.kind { + UntypedKind::Parameter(sym) => { + if let Some(addr) = self.declare_variable(sym, node.identity.clone(), kind, diag) { + self.make_node( + node.identity.clone(), + BoundKind::Define { + name: sym.clone(), + addr, + kind, + value: Box::new(self.make_node(node.identity.clone(), BoundKind::Nop)), + captured_by: Vec::new(), // Filled by post-pass + }, + ) + } else { + self.make_node(node.identity.clone(), BoundKind::Error) + } + } + UntypedKind::Tuple { elements } => { + let mut bound_elems = Vec::new(); + for e in elements { + bound_elems.push(self.bind_pattern(e, kind, diag)); + } + self.make_node( + node.identity.clone(), + BoundKind::Tuple { + elements: bound_elems, + }, + ) + } + _ => { + diag.push_error( + format!("Invalid node in pattern: {:?}", node.kind), + Some(node.identity.clone()), + ); + self.make_node(node.identity.clone(), BoundKind::Error) + } + } + } + + fn bind_assign_pattern( + &mut self, + node: &Node, + diag: &mut Diagnostics, + ) -> BoundNode { + match &node.kind { + UntypedKind::Identifier(sym) => { + if let Some(addr) = self.resolve_variable(sym, diag, &node.identity) { + self.make_node( + node.identity.clone(), + BoundKind::Set { + addr, + value: Box::new(self.make_node(node.identity.clone(), BoundKind::Nop)), + }, + ) + } else { + self.make_node(node.identity.clone(), BoundKind::Error) + } + } + UntypedKind::Tuple { elements } => { + let mut bound_elems = Vec::new(); + for e in elements { + bound_elems.push(self.bind_assign_pattern(e, diag)); + } + self.make_node( + node.identity.clone(), + BoundKind::Tuple { + elements: bound_elems, + }, + ) + } + _ => { + diag.push_error( + format!("Invalid node in assignment pattern: {:?}", node.kind), + Some(node.identity.clone()), + ); + self.make_node(node.identity.clone(), BoundKind::Error) + } + } + } + + fn make_node(&self, identity: Identity, kind: BoundKind<()>) -> BoundNode { + Node { + identity, + kind, + ty: (), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::diagnostics::Diagnostics; + use crate::ast::parser::Parser; + + #[test] + fn test_upvalue_capture_sets_is_boxed() { + // Wrap in a lambda to ensure 'x' is a local variable, not a global + let source = "(fn [] (do (def x 10) (def f (fn [] x)) x))"; + let mut parser = Parser::new(source); + let untyped = parser.parse_expression(); + + let globals = Rc::new(RefCell::new(HashMap::new())); + let mut diagnostics = Diagnostics::new(); + let (bound, captures) = Binder::bind_root(globals, &untyped, &mut diagnostics).unwrap(); + + // Structure: Lambda -> Block -> [ Define(x), Define(f), Get(x) ] + if let BoundKind::Lambda { body, .. } = &bound.kind { + if let BoundKind::Block { exprs } = &body.kind { + let x_decl = &exprs[0]; + if let BoundKind::Define { addr, .. } = &x_decl.kind { + assert!(matches!(addr, Address::Local(_))); + assert!( + captures.contains_key(&x_decl.identity), + "Variable 'x' should have capturers because it is used in lambda 'f'" + ); + } else { + panic!( + "First expression in block should be Define, got {:?}", + x_decl.kind + ); + } + } else { + panic!("Lambda body should be a Block, got {:?}", body.kind); + } + } else { + panic!("Root should be a Lambda, got {:?}", bound.kind); + } + } + + #[test] + fn test_no_capture_not_boxed() { + let source = "(fn [] (do (def x 10) x))"; + let mut parser = Parser::new(source); + let untyped = parser.parse_expression(); + + let globals = Rc::new(RefCell::new(HashMap::new())); + let mut diagnostics = Diagnostics::new(); + let (bound, captures) = Binder::bind_root(globals, &untyped, &mut diagnostics).unwrap(); + + if let BoundKind::Lambda { body, .. } = &bound.kind { + if let BoundKind::Block { exprs } = &body.kind { + let x_decl = &exprs[0]; + if let BoundKind::Define { addr, .. } = &x_decl.kind { + assert!(matches!(addr, Address::Local(_))); + assert!( + !captures.contains_key(&x_decl.identity), + "Variable 'x' should NOT have any capturers" + ); + } else { + panic!("First expression should be Define"); + } + } else { + panic!("Lambda body should be a Block"); + } + } else { + panic!("Root should be a Lambda"); + } + } + + #[test] + fn test_redefinition_error() { + let source = "(do (def x 1) (def x 2))"; + let mut parser = Parser::new(source); + let untyped = parser.parse_expression(); + + let globals = Rc::new(RefCell::new(HashMap::new())); + let mut diagnostics = Diagnostics::new(); + let _ = Binder::bind_root(globals, &untyped, &mut diagnostics); + + assert!(diagnostics.has_errors()); + assert!(diagnostics.items[0].message.contains("already defined")); + } + + #[test] + fn test_repro_global_redefinition() { + let globals = Rc::new(RefCell::new(HashMap::new())); + + // First run: defines 'x' + let source1 = "(def x 1)"; + let untyped1 = Parser::new(source1).parse_expression(); + let mut diagnostics = Diagnostics::new(); + assert!(Binder::bind_root(globals.clone(), &untyped1, &mut diagnostics).is_ok()); + + // Second run: attempts to redefine 'x' in the same global environment + let source2 = "(def x 2)"; + let untyped2 = Parser::new(source2).parse_expression(); + let mut diagnostics2 = Diagnostics::new(); + let _ = Binder::bind_root(globals.clone(), &untyped2, &mut diagnostics2); + + assert!(diagnostics2.has_errors()); + assert!(diagnostics2.items[0].message.contains("already defined")); + } +} diff --git a/src/ast/compiler/bound_nodes.rs b/src/ast/compiler/bound_nodes.rs index 725ab91..e7fbae9 100644 --- a/src/ast/compiler/bound_nodes.rs +++ b/src/ast/compiler/bound_nodes.rs @@ -206,9 +206,10 @@ where }, ) => na == nb && aa == ab && ka == kb && va == vb && ca == cb, (BoundKind::FieldAccessor(a), BoundKind::FieldAccessor(b)) => a == b, - (BoundKind::GetField { rec: ra, field: fa }, BoundKind::GetField { rec: rb, field: fb }) => { - ra == rb && fa == fb - } + ( + BoundKind::GetField { rec: ra, field: fa }, + BoundKind::GetField { rec: rb, field: fb }, + ) => ra == rb && fa == fb, ( BoundKind::If { cond: ca, @@ -258,9 +259,16 @@ where (BoundKind::Again { args: aa }, BoundKind::Again { args: ab }) => aa == ab, (BoundKind::Block { exprs: ea }, BoundKind::Block { exprs: eb }) => ea == eb, (BoundKind::Tuple { elements: ea }, BoundKind::Tuple { elements: eb }) => ea == eb, - (BoundKind::Record { layout: la, values: va }, BoundKind::Record { layout: lb, values: vb }) => { - std::sync::Arc::ptr_eq(la, lb) && va == vb - } + ( + BoundKind::Record { + layout: la, + values: va, + }, + BoundKind::Record { + layout: lb, + values: vb, + }, + ) => std::sync::Arc::ptr_eq(la, lb) && va == vb, ( BoundKind::Expansion { original_call: ca, diff --git a/src/ast/compiler/captures.rs b/src/ast/compiler/captures.rs index 138edac..fc9aae2 100644 --- a/src/ast/compiler/captures.rs +++ b/src/ast/compiler/captures.rs @@ -89,7 +89,11 @@ impl CapturePass { args: Box::new(Self::transform(*args, capture_map)), }; } - BoundKind::Pipe { inputs, lambda, out_type } => { + BoundKind::Pipe { + inputs, + lambda, + out_type, + } => { let mut t_inputs = Vec::with_capacity(inputs.len()); for input in inputs { t_inputs.push(Self::transform(input, capture_map)); diff --git a/src/ast/compiler/dumper.rs b/src/ast/compiler/dumper.rs index 2dc767d..26cbb58 100644 --- a/src/ast/compiler/dumper.rs +++ b/src/ast/compiler/dumper.rs @@ -66,9 +66,7 @@ impl Dumper { self.log(&format!("Get: {} ({:?})", name.name, addr), node) } - BoundKind::FieldAccessor(k) => { - self.log(&format!("FieldAccessor: .{}", k.name()), node) - } + BoundKind::FieldAccessor(k) => self.log(&format!("FieldAccessor: .{}", k.name()), node), BoundKind::GetField { rec, field } => { self.log(&format!("GetField: .{}", field.name()), node); @@ -232,7 +230,10 @@ impl Dumper { } BoundKind::Record { layout, values } => { - self.log(&format!("Record (Layout: {} fields)", layout.fields.len()), node); + self.log( + &format!("Record (Layout: {} fields)", layout.fields.len()), + node, + ); self.indent += 1; for v in values { self.visit(v); diff --git a/src/ast/compiler/optimizer/engine.rs b/src/ast/compiler/optimizer/engine.rs index dd20c11..4f240bd 100644 --- a/src/ast/compiler/optimizer/engine.rs +++ b/src/ast/compiler/optimizer/engine.rs @@ -386,7 +386,11 @@ impl Optimizer { ) } - BoundKind::Pipe { inputs, lambda, out_type } => { + BoundKind::Pipe { + inputs, + lambda, + out_type, + } => { let mut o_inputs = Vec::with_capacity(inputs.len()); for input in inputs { o_inputs.push(self.visit_node(input, sub, path)); @@ -559,17 +563,28 @@ impl Optimizer { .collect(); (BoundKind::Tuple { elements }, node.ty.clone()) } - BoundKind::Record { ref layout, ref values } => { + BoundKind::Record { + ref layout, + ref values, + } => { let mapped_values: Vec<_> = values .iter() .map(|v| self.visit_node(v.clone(), sub, path)) .collect(); - - if self.enabled && let Some(folded) = folder.try_fold_record(layout, &mapped_values, &node) { + + if self.enabled + && let Some(folded) = folder.try_fold_record(layout, &mapped_values, &node) + { return folded; } - - (BoundKind::Record { layout: layout.clone(), values: mapped_values }, node.ty.clone()) + + ( + BoundKind::Record { + layout: layout.clone(), + values: mapped_values, + }, + node.ty.clone(), + ) } BoundKind::Expansion { ref original_call, diff --git a/src/ast/compiler/optimizer/folder.rs b/src/ast/compiler/optimizer/folder.rs index d126032..1f3d186 100644 --- a/src/ast/compiler/optimizer/folder.rs +++ b/src/ast/compiler/optimizer/folder.rs @@ -1,6 +1,6 @@ use crate::ast::compiler::bound_nodes::{Address, AnalyzedNode, BoundKind, NodeMetrics}; use crate::ast::nodes::Node; -use crate::ast::types::{Purity, StaticType, Value, RecordLayout}; +use crate::ast::types::{Purity, RecordLayout, StaticType, Value}; use std::cell::RefCell; use std::rc::Rc; diff --git a/src/ast/compiler/optimizer/utils.rs b/src/ast/compiler/optimizer/utils.rs index 6b6486d..97cd27a 100644 --- a/src/ast/compiler/optimizer/utils.rs +++ b/src/ast/compiler/optimizer/utils.rs @@ -158,7 +158,10 @@ impl UsageInfo { } self.collect(lambda); } - BoundKind::Nop | BoundKind::FieldAccessor(_) | BoundKind::Extension(_) | BoundKind::Error => {} + BoundKind::Nop + | BoundKind::FieldAccessor(_) + | BoundKind::Extension(_) + | BoundKind::Error => {} } } diff --git a/src/ast/compiler/specializer.rs b/src/ast/compiler/specializer.rs index 75c7f7a..ab5983e 100644 --- a/src/ast/compiler/specializer.rs +++ b/src/ast/compiler/specializer.rs @@ -132,10 +132,7 @@ impl Specializer { (BoundKind::Tuple { elements }, node.ty.clone()) } BoundKind::Record { layout, values } => { - let values = values - .into_iter() - .map(|v| self.visit_node(v)) - .collect(); + let values = values.into_iter().map(|v| self.visit_node(v)).collect(); (BoundKind::Record { layout, values }, node.ty.clone()) } BoundKind::Expansion { diff --git a/src/ast/compiler/tco.rs b/src/ast/compiler/tco.rs index 7affe57..88e1943 100644 --- a/src/ast/compiler/tco.rs +++ b/src/ast/compiler/tco.rs @@ -48,7 +48,11 @@ impl TCO { args: Box::new(Self::transform(Rc::new((**args).clone()), false)), } } - BoundKind::Pipe { inputs, lambda, out_type } => { + BoundKind::Pipe { + inputs, + lambda, + out_type, + } => { let mut t_inputs = Vec::with_capacity(inputs.len()); for input in inputs { t_inputs.push(Self::transform(Rc::new(input.clone()), false)); diff --git a/src/ast/compiler/type_checker.rs b/src/ast/compiler/type_checker.rs index a78d979..369ff0c 100644 --- a/src/ast/compiler/type_checker.rs +++ b/src/ast/compiler/type_checker.rs @@ -79,7 +79,12 @@ impl TypeChecker { Self { global_types } } - pub fn check(&self, node: BoundNode, arg_types: &[StaticType], diag: &mut Diagnostics) -> TypedNode { + pub fn check( + &self, + node: BoundNode, + arg_types: &[StaticType], + diag: &mut Diagnostics, + ) -> TypedNode { match node.kind { BoundKind::Lambda { params, @@ -105,7 +110,12 @@ impl TypeChecker { StaticType::Tuple(arg_types.to_vec()) }; - let params_typed = self.check_params(params.as_ref().clone(), &arg_tuple_ty, &mut lambda_ctx, diag); + let params_typed = self.check_params( + params.as_ref().clone(), + &arg_tuple_ty, + &mut lambda_ctx, + diag, + ); // 4. Check body with the new types let body_typed = self.check_node((*body).clone(), &mut lambda_ctx, diag); @@ -212,7 +222,13 @@ impl TypeChecker { | StaticType::Record(_) | StaticType::Error => {} _ => { - diag.push_error(format!("Cannot destructure type {} as a tuple/vector", specialized_ty), Some(node.identity.clone())); + diag.push_error( + format!( + "Cannot destructure type {} as a tuple/vector", + specialized_ty + ), + Some(node.identity.clone()), + ); return Node { identity: node.identity, kind: BoundKind::Error, @@ -251,7 +267,10 @@ impl TypeChecker { } BoundKind::Error => (BoundKind::Error, StaticType::Error), _ => { - diag.push_error("Invalid node in parameter list", Some(node.identity.clone())); + diag.push_error( + "Invalid node in parameter list", + Some(node.identity.clone()), + ); (BoundKind::Error, StaticType::Error) } }; @@ -263,7 +282,12 @@ impl TypeChecker { } } - fn check_node(&self, node: BoundNode, ctx: &mut TypeContext, diag: &mut Diagnostics) -> TypedNode { + fn check_node( + &self, + node: BoundNode, + ctx: &mut TypeContext, + diag: &mut Diagnostics, + ) -> TypedNode { let (kind, ty) = match node.kind { BoundKind::Nop => (BoundKind::Nop, StaticType::Void), @@ -306,7 +330,9 @@ impl TypeChecker { ) } - BoundKind::FieldAccessor(k) => (BoundKind::FieldAccessor(k), StaticType::FieldAccessor(k)), + BoundKind::FieldAccessor(k) => { + (BoundKind::FieldAccessor(k), StaticType::FieldAccessor(k)) + } BoundKind::GetField { rec, field } => { let rec_typed = self.check_node(*rec, ctx, diag); @@ -315,14 +341,24 @@ impl TypeChecker { if let Some(idx) = layout.index_of(field) { layout.fields[idx].1.clone() } else { - diag.push_error(format!("Record does not have field :{}", field.name()), Some(rec_typed.identity.clone())); + diag.push_error( + format!("Record does not have field :{}", field.name()), + Some(rec_typed.identity.clone()), + ); StaticType::Error } } StaticType::Any => StaticType::Any, StaticType::Error => StaticType::Error, _ => { - diag.push_error(format!("Cannot access field :{} on non-record type {}", field.name(), rec_typed.ty), Some(rec_typed.identity.clone())); + diag.push_error( + format!( + "Cannot access field :{} on non-record type {}", + field.name(), + rec_typed.ty + ), + Some(rec_typed.identity.clone()), + ); StaticType::Error } }; @@ -463,7 +499,12 @@ impl TypeChecker { TypeContext::new(64, upvalue_types, ctx.global_types, Some(ctx)); // 3. Check parameters and body - let params_typed = self.check_params(params.as_ref().clone(), &StaticType::Any, &mut lambda_ctx, diag); + let params_typed = self.check_params( + params.as_ref().clone(), + &StaticType::Any, + &mut lambda_ctx, + diag, + ); // Set current params type for 'again' validation lambda_ctx.current_params_ty = Some(params_typed.ty.clone()); @@ -515,7 +556,13 @@ impl TypeChecker { let ret_ty = match callee_typed.ty.resolve_call(&args_typed.ty) { Some(ty) => ty, None => { - diag.push_error(format!("Invalid arguments for function call. Expected {}, got {}", callee_typed.ty, args_typed.ty), Some(node.identity.clone())); + diag.push_error( + format!( + "Invalid arguments for function call. Expected {}, got {}", + callee_typed.ty, args_typed.ty + ), + Some(node.identity.clone()), + ); StaticType::Error } }; @@ -553,7 +600,13 @@ impl TypeChecker { if let Some(expected_ty) = &ctx.current_params_ty && !expected_ty.is_assignable_from(&args_typed.ty) { - diag.push_error(format!("Type mismatch in 'again' call: expected {}, but got {}", expected_ty, args_typed.ty), Some(node.identity.clone())); + diag.push_error( + format!( + "Type mismatch in 'again' call: expected {}, but got {}", + expected_ty, args_typed.ty + ), + Some(node.identity.clone()), + ); } ( @@ -606,13 +659,13 @@ impl TypeChecker { BoundKind::Record { layout, values } => { let mut typed_values = Vec::with_capacity(values.len()); let mut fields_ty = Vec::with_capacity(values.len()); - + for (i, v) in values.into_iter().enumerate() { let vt = self.check_node(v, ctx, diag); fields_ty.push((layout.fields[i].0, vt.ty.clone())); typed_values.push(vt); } - + let new_layout = crate::ast::types::RecordLayout::get_or_create(fields_ty); ( BoundKind::Record { @@ -639,7 +692,13 @@ impl TypeChecker { } BoundKind::Extension(_ext) => { - diag.push_error(format!("TypeChecking for extension '{}' not implemented", _ext.display_name()), Some(node.identity.clone())); + diag.push_error( + format!( + "TypeChecking for extension '{}' not implemented", + _ext.display_name() + ), + Some(node.identity.clone()), + ); (BoundKind::Error, StaticType::Error) } BoundKind::Error => (BoundKind::Error, StaticType::Error), diff --git a/src/ast/environment.rs b/src/ast/environment.rs index 3173337..05fe95e 100644 --- a/src/ast/environment.rs +++ b/src/ast/environment.rs @@ -36,7 +36,7 @@ impl CompilationResult { diagnostics: Diagnostics::new(), } } - + pub fn error(msg: impl Into) -> Self { let mut diag = Diagnostics::new(); diag.push_error(msg, None); @@ -123,11 +123,16 @@ impl MacroEvaluator for RuntimeMacroEvaluator { let bound_ast = crate::ast::compiler::captures::CapturePass::apply(bound_ast, &captures); let checker = TypeChecker::new(self.global_types.clone()); let typed_ast = checker.check(bound_ast, &[], &mut diag); - + if diag.has_errors() { - return Err(diag.items.into_iter().map(|d| d.message).collect::>().join("\n")); + return Err(diag + .items + .into_iter() + .map(|d| d.message) + .collect::>() + .join("\n")); } - + let exec_ast = TCO::optimize(Analyzer::analyze(&typed_ast, &HashMap::new())); // Minimal analysis for macro eval let mut vm = VM::new(self.global_values.clone()); @@ -191,7 +196,9 @@ impl Environment { let mut parser = crate::ast::parser::Parser::new(source); let untyped_ast = parser.parse_expression(); let mut expander = self.get_expander(); - let _ = expander.expand(untyped_ast).expect("Failed to expand prelude"); + let _ = expander + .expand(untyped_ast) + .expect("Failed to expand prelude"); *self.macro_registry.borrow_mut() = expander.into_registry(); } @@ -261,7 +268,9 @@ impl Environment { let untyped_ast = parser.parse_expression(); if !parser.at_eof() { - parser.diagnostics.push_error("Unexpected trailing expressions in script.", None); + parser + .diagnostics + .push_error("Unexpected trailing expressions in script.", None); return CompilationResult { ast: None, diagnostics: parser.diagnostics, @@ -273,17 +282,24 @@ impl Environment { Ok(ast) => ast, Err(e) => { diagnostics.push_error(e, None); - return CompilationResult { ast: None, diagnostics }; + return CompilationResult { + ast: None, + diagnostics, + }; } }; - let (bound_ast, captures) = match Binder::bind_root(self.global_names.clone(), &expanded_ast, &mut diagnostics) { - Ok(res) => res, - Err(e) => { - diagnostics.push_error(e, None); - return CompilationResult { ast: None, diagnostics }; - } - }; + let (bound_ast, captures) = + match Binder::bind_root(self.global_names.clone(), &expanded_ast, &mut diagnostics) { + Ok(res) => res, + Err(e) => { + diagnostics.push_error(e, None); + return CompilationResult { + ast: None, + diagnostics, + }; + } + }; let bound_ast = crate::ast::compiler::captures::CapturePass::apply(bound_ast, &captures); @@ -373,7 +389,10 @@ impl Environment { let mut final_res = res; if let Value::Object(obj) = &final_res - && obj.as_any().downcast_ref::().is_some() + && obj + .as_any() + .downcast_ref::() + .is_some() { final_res = match vm.run_with_args(obj.clone(), args) { Ok(v) => v, @@ -407,9 +426,14 @@ impl Environment { let mut diag = Diagnostics::new(); let checker = TypeChecker::new(global_types.clone()); let retyped_ast = checker.check(func_template, arg_types, &mut diag); - + if diag.has_errors() { - return Err(diag.items.into_iter().map(|d| d.message).collect::>().join("\n")); + return Err(diag + .items + .into_iter() + .map(|d| d.message) + .collect::>() + .join("\n")); } let analyzed = Analyzer::analyze(&retyped_ast, &global_purity.borrow()); @@ -465,7 +489,9 @@ impl Environment { } res } else { - self.compile(source).into_result().and_then(|ast| self.run_script_compiled(ast)) + self.compile(source) + .into_result() + .and_then(|ast| self.run_script_compiled(ast)) } } @@ -492,7 +518,11 @@ impl Environment { while let Ok(Value::TailCallRequest(payload)) = result { let (next_obj, next_args) = *payload; - if next_obj.as_any().downcast_ref::().is_some() { + if next_obj + .as_any() + .downcast_ref::() + .is_some() + { result = vm.run_with_args_observed(&mut observer, next_obj, next_args); } else { result = Err(format!( @@ -502,9 +532,9 @@ impl Environment { break; } } - + self.run_pipeline(); - + Ok((result, observer.logs)) } } diff --git a/src/ast/parser.rs b/src/ast/parser.rs index e3589a7..68cd672 100644 --- a/src/ast/parser.rs +++ b/src/ast/parser.rs @@ -1,7 +1,7 @@ +use crate::ast::diagnostics::Diagnostics; use crate::ast::lexer::{Lexer, Token, TokenKind}; use crate::ast::nodes::{Node, Symbol, UntypedKind}; use crate::ast::types::{Identity, Keyword, NodeIdentity, Value}; -use crate::ast::diagnostics::Diagnostics; use std::rc::Rc; pub struct Parser<'a> { @@ -35,7 +35,8 @@ impl<'a> Parser<'a> { let next = match self.lexer.next_token() { Ok(t) => t, Err(e) => { - self.diagnostics.push_error(e, Some(NodeIdentity::new(self.current_token.location))); + self.diagnostics + .push_error(e, Some(NodeIdentity::new(self.current_token.location))); Token { kind: TokenKind::EOF, location: self.current_token.location, @@ -331,7 +332,10 @@ impl<'a> Parser<'a> { fn parse_param_vector(&mut self) -> Node { if *self.peek() != TokenKind::LeftBracket { self.diagnostics.push_error( - format!("Expected parameter vector [...] for fn, found {:?}", self.peek()), + format!( + "Expected parameter vector [...] for fn, found {:?}", + self.peek() + ), Some(NodeIdentity::new(self.current_token.location)), ); return Node { @@ -376,7 +380,10 @@ impl<'a> Parser<'a> { } _ => { self.diagnostics.push_error( - format!("Expected identifier or pattern vector [...] for definition, found {:?}", next), + format!( + "Expected identifier or pattern vector [...] for definition, found {:?}", + next + ), Some(NodeIdentity::new(self.current_token.location)), ); Node { @@ -388,11 +395,7 @@ impl<'a> Parser<'a> { } } - fn parse_call( - &mut self, - callee: Node, - identity: Identity, - ) -> Node { + fn parse_call(&mut self, callee: Node, identity: Identity) -> Node { let mut elements = Vec::new(); while *self.peek() != TokenKind::RightParen && *self.peek() != TokenKind::EOF { diff --git a/src/ast/rtl/math.rs b/src/ast/rtl/math.rs index d76e978..25c18b2 100644 --- a/src/ast/rtl/math.rs +++ b/src/ast/rtl/math.rs @@ -1,173 +1,176 @@ -use crate::ast::environment::Environment; -use crate::ast::types::{Purity, Signature, StaticType, Value}; -use std::f64::consts; - -pub fn register(env: &Environment) { - // Constants - env.register_constant("PI", StaticType::Float, Value::Float(consts::PI)); - env.register_constant("E", StaticType::Float, Value::Float(consts::E)); - - // Helper to get f64 from Value (Int or Float) - fn to_f64(v: &Value) -> Option { - match v { - Value::Float(f) => Some(*f), - Value::Int(i) => Some(*i as f64), - _ => None, - } - } - - // Unary functions (float -> float) - let register_unary = |name: &'static str, f: fn(f64) -> f64| { - let ty = StaticType::Function(Box::new(Signature { - params: StaticType::Tuple(vec![StaticType::Float]), - ret: StaticType::Float, - })); - env.register_native_fn(name, ty, Purity::Pure, move |args| { - if let Some(x) = to_f64(&args[0]) { - Value::Float(f(x)) - } else { - Value::Void - } - }); - }; - - // Unary functions (float -> int) - let register_to_int = |name: &'static str, f: fn(f64) -> f64| { - let ty = StaticType::Function(Box::new(Signature { - params: StaticType::Tuple(vec![StaticType::Float]), - ret: StaticType::Int, - })); - env.register_native_fn(name, ty, Purity::Pure, move |args| { - if let Some(x) = to_f64(&args[0]) { - Value::Int(f(x) as i64) - } else { - Value::Void - } - }); - }; - - register_unary("sqrt", f64::sqrt); - register_unary("sin", f64::sin); - register_unary("cos", f64::cos); - register_unary("tan", f64::tan); - register_unary("asin", f64::asin); - register_unary("acos", f64::acos); - register_unary("atan", f64::atan); - register_unary("exp", f64::exp); - register_unary("ln", f64::ln); - register_unary("log10", f64::log10); - register_unary("fract", f64::fract); - - // Rounding functions returning Int - register_to_int("round", f64::round); - register_to_int("floor", f64::floor); - register_to_int("ceil", f64::ceil); - register_to_int("trunc", f64::trunc); - - // Binary functions (float, float -> float) - let register_binary = |name: &'static str, f: fn(f64, f64) -> f64| { - let ty = StaticType::Function(Box::new(Signature { - params: StaticType::Tuple(vec![StaticType::Float, StaticType::Float]), - ret: StaticType::Float, - })); - env.register_native_fn(name, ty, Purity::Pure, move |args| { - if let (Some(a), Some(b)) = (to_f64(&args[0]), to_f64(&args[1])) { - Value::Float(f(a, b)) - } else { - Value::Void - } - }); - }; - - register_binary("atan2", f64::atan2); - register_binary("pow", f64::powf); - register_binary("log", f64::log); - - // Specialized abs to handle Int -> Int, Float -> Float - let abs_ty = StaticType::Function(Box::new(Signature { - params: StaticType::Tuple(vec![StaticType::Any]), - ret: StaticType::Any, - })); - env.register_native_fn("abs", abs_ty, Purity::Pure, |args| { - if args.len() != 1 { - return Value::Void; - } - match args[0] { - Value::Int(i) => Value::Int(i.abs()), - Value::Float(f) => Value::Float(f.abs()), - _ => Value::Void, - } - }); - - // Variadic min / max - let variadic_ty = StaticType::Function(Box::new(Signature { - params: StaticType::Any, - ret: StaticType::Any, - })); - - env.register_native_fn("min", variadic_ty.clone(), Purity::Pure, |args| { - if args.is_empty() { - return Value::Void; - } - let mut best = args[0].clone(); - for arg in &args[1..] { - if let Some(ord) = arg.partial_cmp(&best) - && ord == std::cmp::Ordering::Less - { - best = arg.clone(); - } - } - best - }); - - env.register_native_fn("max", variadic_ty, Purity::Pure, |args| { - if args.is_empty() { - return Value::Void; - } - let mut best = args[0].clone(); - for arg in &args[1..] { - if let Some(ord) = arg.partial_cmp(&best) - && ord == std::cmp::Ordering::Greater - { - best = arg.clone(); - } - } - best - }); - - // Random generator factory (Closure-based) - let generator_sig = Signature { - params: StaticType::Tuple(vec![]), - ret: StaticType::Float, - }; - - let make_random_ty = StaticType::FunctionOverloads(vec![ - Signature { - params: StaticType::Tuple(vec![]), - ret: StaticType::Function(Box::new(generator_sig.clone())), - }, - Signature { - params: StaticType::Tuple(vec![StaticType::Int]), - ret: StaticType::Function(Box::new(generator_sig)), - }, - ]); - - env.register_native_fn("make-random", make_random_ty, Purity::SideEffectFree, |args| { - let rng = if args.is_empty() { - fastrand::Rng::new() - } else if let Value::Int(s) = args[0] { - fastrand::Rng::with_seed(s as u64) - } else { - fastrand::Rng::new() - }; - - let prng = std::rc::Rc::new(std::cell::RefCell::new(rng)); - - Value::Function(std::rc::Rc::new(crate::ast::types::NativeFunction { - func: std::rc::Rc::new(move |_| { - Value::Float(prng.borrow_mut().f64()) - }), - purity: Purity::Impure, - })) - }); -} +use crate::ast::environment::Environment; +use crate::ast::types::{Purity, Signature, StaticType, Value}; +use std::f64::consts; + +pub fn register(env: &Environment) { + // Constants + env.register_constant("PI", StaticType::Float, Value::Float(consts::PI)); + env.register_constant("E", StaticType::Float, Value::Float(consts::E)); + + // Helper to get f64 from Value (Int or Float) + fn to_f64(v: &Value) -> Option { + match v { + Value::Float(f) => Some(*f), + Value::Int(i) => Some(*i as f64), + _ => None, + } + } + + // Unary functions (float -> float) + let register_unary = |name: &'static str, f: fn(f64) -> f64| { + let ty = StaticType::Function(Box::new(Signature { + params: StaticType::Tuple(vec![StaticType::Float]), + ret: StaticType::Float, + })); + env.register_native_fn(name, ty, Purity::Pure, move |args| { + if let Some(x) = to_f64(&args[0]) { + Value::Float(f(x)) + } else { + Value::Void + } + }); + }; + + // Unary functions (float -> int) + let register_to_int = |name: &'static str, f: fn(f64) -> f64| { + let ty = StaticType::Function(Box::new(Signature { + params: StaticType::Tuple(vec![StaticType::Float]), + ret: StaticType::Int, + })); + env.register_native_fn(name, ty, Purity::Pure, move |args| { + if let Some(x) = to_f64(&args[0]) { + Value::Int(f(x) as i64) + } else { + Value::Void + } + }); + }; + + register_unary("sqrt", f64::sqrt); + register_unary("sin", f64::sin); + register_unary("cos", f64::cos); + register_unary("tan", f64::tan); + register_unary("asin", f64::asin); + register_unary("acos", f64::acos); + register_unary("atan", f64::atan); + register_unary("exp", f64::exp); + register_unary("ln", f64::ln); + register_unary("log10", f64::log10); + register_unary("fract", f64::fract); + + // Rounding functions returning Int + register_to_int("round", f64::round); + register_to_int("floor", f64::floor); + register_to_int("ceil", f64::ceil); + register_to_int("trunc", f64::trunc); + + // Binary functions (float, float -> float) + let register_binary = |name: &'static str, f: fn(f64, f64) -> f64| { + let ty = StaticType::Function(Box::new(Signature { + params: StaticType::Tuple(vec![StaticType::Float, StaticType::Float]), + ret: StaticType::Float, + })); + env.register_native_fn(name, ty, Purity::Pure, move |args| { + if let (Some(a), Some(b)) = (to_f64(&args[0]), to_f64(&args[1])) { + Value::Float(f(a, b)) + } else { + Value::Void + } + }); + }; + + register_binary("atan2", f64::atan2); + register_binary("pow", f64::powf); + register_binary("log", f64::log); + + // Specialized abs to handle Int -> Int, Float -> Float + let abs_ty = StaticType::Function(Box::new(Signature { + params: StaticType::Tuple(vec![StaticType::Any]), + ret: StaticType::Any, + })); + env.register_native_fn("abs", abs_ty, Purity::Pure, |args| { + if args.len() != 1 { + return Value::Void; + } + match args[0] { + Value::Int(i) => Value::Int(i.abs()), + Value::Float(f) => Value::Float(f.abs()), + _ => Value::Void, + } + }); + + // Variadic min / max + let variadic_ty = StaticType::Function(Box::new(Signature { + params: StaticType::Any, + ret: StaticType::Any, + })); + + env.register_native_fn("min", variadic_ty.clone(), Purity::Pure, |args| { + if args.is_empty() { + return Value::Void; + } + let mut best = args[0].clone(); + for arg in &args[1..] { + if let Some(ord) = arg.partial_cmp(&best) + && ord == std::cmp::Ordering::Less + { + best = arg.clone(); + } + } + best + }); + + env.register_native_fn("max", variadic_ty, Purity::Pure, |args| { + if args.is_empty() { + return Value::Void; + } + let mut best = args[0].clone(); + for arg in &args[1..] { + if let Some(ord) = arg.partial_cmp(&best) + && ord == std::cmp::Ordering::Greater + { + best = arg.clone(); + } + } + best + }); + + // Random generator factory (Closure-based) + let generator_sig = Signature { + params: StaticType::Tuple(vec![]), + ret: StaticType::Float, + }; + + let make_random_ty = StaticType::FunctionOverloads(vec![ + Signature { + params: StaticType::Tuple(vec![]), + ret: StaticType::Function(Box::new(generator_sig.clone())), + }, + Signature { + params: StaticType::Tuple(vec![StaticType::Int]), + ret: StaticType::Function(Box::new(generator_sig)), + }, + ]); + + env.register_native_fn( + "make-random", + make_random_ty, + Purity::SideEffectFree, + |args| { + let rng = if args.is_empty() { + fastrand::Rng::new() + } else if let Value::Int(s) = args[0] { + fastrand::Rng::with_seed(s as u64) + } else { + fastrand::Rng::new() + }; + + let prng = std::rc::Rc::new(std::cell::RefCell::new(rng)); + + Value::Function(std::rc::Rc::new(crate::ast::types::NativeFunction { + func: std::rc::Rc::new(move |_| Value::Float(prng.borrow_mut().f64())), + purity: Purity::Impure, + })) + }, + ); +} diff --git a/src/ast/rtl/mod.rs b/src/ast/rtl/mod.rs index 0f7535f..1a0cd78 100644 --- a/src/ast/rtl/mod.rs +++ b/src/ast/rtl/mod.rs @@ -14,4 +14,4 @@ pub fn register(env: &Environment) { math::register(env); series::register(env); streams::register(env); -} \ No newline at end of file +} diff --git a/src/ast/rtl/series.rs b/src/ast/rtl/series.rs index 65b0011..f08cf26 100644 --- a/src/ast/rtl/series.rs +++ b/src/ast/rtl/series.rs @@ -1,530 +1,558 @@ -use std::any::Any; -use std::collections::VecDeque; -use std::fmt::{self, Debug}; -use std::rc::Rc; -use std::sync::Arc; - -use crate::ast::types::{Keyword, Object, RecordLayout, StaticType, Value}; - -// ============================================================================ -// 1. Core Storage Engine -// ============================================================================ - -/// A generic, efficient ring buffer for time series data. -/// (Replaces Delphi's TChunkArray for now with VecDeque for O(1) push/pop) -#[derive(Clone)] -pub struct RingBuffer { - buffer: VecDeque, - total_count: u64, -} - -impl Default for RingBuffer { - fn default() -> Self { - Self::new() - } -} - -impl RingBuffer { - pub fn new() -> Self { - Self { - buffer: VecDeque::new(), - total_count: 0, - } - } - - /// Adds an element. Removes the oldest if the lookback limit is reached. - #[inline] - pub fn push(&mut self, item: T, lookback: Option) { - self.buffer.push_back(item); - self.total_count += 1; - - if let Some(limit) = lookback { - while self.buffer.len() > limit { - self.buffer.pop_front(); - } - } - } - - /// Access in "Financial Style": Index 0 is the newest element. - #[inline] - pub fn get(&self, index: usize) -> Option<&T> { - let len = self.buffer.len(); - if index < len { - self.buffer.get(len - 1 - index) - } else { - None - } - } - - #[inline] - pub fn total_count(&self) -> u64 { - self.total_count - } - - #[inline] - pub fn len(&self) -> usize { - self.buffer.len() - } - - pub fn is_empty(&self) -> bool { - self.buffer.is_empty() - } -} - -// ============================================================================ -// 2. Trait & Primitives for Fast-Paths -// ============================================================================ - -/// Marker trait ensuring our fast arrays only store true, flat scalars -/// (No pointers/heaps like String or Record). -pub trait ScalarValue: Copy + Debug + 'static {} - -impl ScalarValue for f64 {} -impl ScalarValue for i64 {} -impl ScalarValue for bool {} - -/// A highly optimized, type-safe series for primitive values (Float, Int, Bool). -#[derive(Clone)] -pub struct ScalarSeries { - pub data: RingBuffer, - pub type_name: &'static str, -} - -impl ScalarSeries { - pub fn new(type_name: &'static str) -> Self { - Self { - data: RingBuffer::new(), - type_name, - } - } -} - -impl Debug for ScalarSeries { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "ScalarSeries<{}>[len: {}]", self.type_name, self.data.len()) - } -} - -// Makes the series usable as a dynamic Object for the VM. -impl Object for ScalarSeries { - fn type_name(&self) -> &'static str { - self.type_name - } - fn as_any(&self) -> &dyn Any { - self - } -} - -// ============================================================================ -// 2b. Fallback Series for non-scalar types -// ============================================================================ - -/// A generic series for all non-scalar types (Strings, Tuples, etc.) or mixed data. -#[derive(Clone)] -pub struct ValueSeries { - pub data: RingBuffer, -} - -impl Default for ValueSeries { - fn default() -> Self { - Self::new() - } -} - -impl ValueSeries { - pub fn new() -> Self { - Self { - data: RingBuffer::new(), - } - } -} - -impl Debug for ValueSeries { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "ValueSeries[len: {}]", self.data.len()) - } -} - -impl Object for ValueSeries { - fn type_name(&self) -> &'static str { - "ValueSeries" - } - fn as_any(&self) -> &dyn Any { - self - } -} - -// ============================================================================ -// 3. RecordSeries (SoA - Struct of Arrays) -// ============================================================================ - -/// An interface allowing iteration over fields of a RecordSeries independently -/// of the exact type (Type Erasure for member arrays). -pub trait SeriesMember: Object { - /// Gets the value at the specified lookback index as a dynamic Value. - fn get_value(&self, index: usize) -> Option; - - /// Pushes a dynamic Value into the series, performing the necessary downcast. - fn push_value(&mut self, value: Value, limit: Option); -} - -impl SeriesMember for ScalarSeries { - fn get_value(&self, index: usize) -> Option { - self.data.get(index).map(|&v| Value::Float(v)) - } - fn push_value(&mut self, value: Value, limit: Option) { - if let Value::Float(v) = value { - self.data.push(v, limit); - } - } -} - -impl SeriesMember for ScalarSeries { - fn get_value(&self, index: usize) -> Option { - self.data.get(index).map(|&v| Value::Int(v)) - } - fn push_value(&mut self, value: Value, limit: Option) { - if let Value::Int(v) = value { - self.data.push(v, limit); - } else if let Value::DateTime(v) = value { - self.data.push(v, limit); - } - } -} - -impl SeriesMember for ScalarSeries { - fn get_value(&self, index: usize) -> Option { - self.data.get(index).map(|&v| Value::Bool(v)) - } - fn push_value(&mut self, value: Value, limit: Option) { - if let Value::Bool(v) = value { - self.data.push(v, limit); - } - } -} - -impl SeriesMember for ValueSeries { - fn get_value(&self, index: usize) -> Option { - self.data.get(index).cloned() - } - fn push_value(&mut self, value: Value, limit: Option) { - self.data.push(value, limit); - } -} - -/// The "Struct of Arrays" implementation for Records (e.g., Ticks). -/// Instead of holding a large array of Records (AoS), this series -/// splits the record and stores each field in a separate (parallel) series. -#[derive(Clone)] -pub struct RecordSeries { - layout: Arc, - /// Each field of the record gets its own series (SoA). - /// We use Rc so we can iterate fields individually, - /// but also pass them as 0-Copy references to indicators! - fields: Vec>>, -} - -impl RecordSeries { - /// Creates a new RecordSeries matching the given layout. - pub fn new(layout: Arc) -> Self { - let mut fields: Vec>> = Vec::with_capacity(layout.fields.len()); - - for (_, static_type) in &layout.fields { - // Automatically select the optimal storage representation based on the field's static type. - let member: Rc> = match static_type { - StaticType::Float => Rc::new(std::cell::RefCell::new(ScalarSeries::::new("FloatSeries"))), - StaticType::Int | StaticType::DateTime => Rc::new(std::cell::RefCell::new(ScalarSeries::::new("IntSeries"))), - StaticType::Bool => Rc::new(std::cell::RefCell::new(ScalarSeries::::new("BoolSeries"))), - // Fallback for everything else (Text, Lists, nested Records without SoA, etc.) - _ => Rc::new(std::cell::RefCell::new(ValueSeries::new())), - }; - fields.push(member); - } - - Self { layout, fields } - } - - /// The magical 0-Copy Field Mapper! - /// Returns the complete series of a single field. - /// Example: `my_ticks.field(Keyword::intern("close"))` - /// returns an `Rc>>`. No copy involved! - pub fn field(&self, key: Keyword) -> Option>> { - self.layout.index_of(key).map(|idx| self.fields[idx].clone()) - } - - /// Dynamically reconstructs the full Record at the given lookback index. - /// This is an O(N) operation where N is the number of fields, meaning it breaks - /// the 0-Copy performance path, but it's essential for idiomatic AST access `(series 0)`. - pub fn get_record(&self, index: usize) -> Option { - if self.fields.is_empty() { - return None; - } - - let mut vals = Vec::with_capacity(self.fields.len()); - for field in &self.fields { - match field.borrow().get_value(index) { - Some(v) => vals.push(v), - None => return None, // Out of bounds or incomplete - } - } - - Some(Value::Record(self.layout.clone(), Rc::new(vals))) - } -} - -impl Debug for RecordSeries { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "RecordSeries[fields: {}]", self.fields.len()) - } -} - -impl Object for RecordSeries { - fn type_name(&self) -> &'static str { - "RecordSeries" - } - fn as_any(&self) -> &dyn Any { - self - } - fn as_series(&self) -> Option<&dyn crate::ast::types::Series> { - Some(self) - } -} - -impl crate::ast::types::Series for RecordSeries { - fn get_item(&self, index: usize) -> Option { - self.get_record(index) - } -} - -// ============================================================================ -// 4. Series View (Safe, 0-Copy access bridge for the VM) -// ============================================================================ - -/// A lightweight, 0-copy wrapper around a specific column (field) of a RecordSeries. -/// -/// Why is this needed? -/// In a SoA (Struct of Arrays) architecture, columns like `open` or `close` -/// are stored as `Rc>` inside the `RecordSeries`. -/// The `RefCell` is crucial for runtime mutability (so the engine can push new ticks). -/// However, the VM's `Value::Object` expects a pure `Rc`. -/// -/// `SeriesView` bridges this gap safely: It holds the shared reference and implements `Object` itself, -/// acting as a fast, read-only "window" into the underlying column data without copying any arrays. -#[derive(Clone)] -pub struct SeriesView { - pub inner: Rc>, - pub field_name: Keyword, -} - -impl SeriesView { - pub fn new(inner: Rc>, field_name: Keyword) -> Self { - Self { inner, field_name } - } -} - -impl Debug for SeriesView { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "SeriesView[field: {}]", self.field_name.name()) - } -} - -impl Object for SeriesView { - fn type_name(&self) -> &'static str { - "SeriesView" - } - - // This allows the VM or specialized extractors to downcast the View back to its concrete type if needed. - fn as_any(&self) -> &dyn Any { - self - } - - fn as_series(&self) -> Option<&dyn crate::ast::types::Series> { - Some(self) - } -} - -impl crate::ast::types::Series for SeriesView { - fn get_item(&self, index: usize) -> Option { - self.inner.borrow().get_value(index) - } -} - -// ============================================================================ -// 4b. Shared Series (Read-Only Reactive Storage) -// ============================================================================ - -/// A read-only series that shares its underlying buffer with the pipeline engine. -/// This is the basis for the "SharedSeries" in the Dual Series Architecture. -#[derive(Clone)] -pub struct SharedSeries { - pub buffer: Rc>>, - pub type_name: &'static str, - pub converter: fn(T) -> Value, -} - -impl Debug for SharedSeries { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "SharedSeries<{}>[len: {}]", self.type_name, self.buffer.borrow().len()) - } -} - -impl Object for SharedSeries { - fn type_name(&self) -> &'static str { - self.type_name - } - fn as_any(&self) -> &dyn Any { - self - } - fn as_series(&self) -> Option<&dyn crate::ast::types::Series> { - Some(self) - } -} - -impl crate::ast::types::Series for SharedSeries { - fn get_item(&self, index: usize) -> Option { - self.buffer.borrow().get(index).map(|&v| (self.converter)(v)) - } -} - -/// A read-only series for generic Values that shares its buffer with the pipeline. -#[derive(Clone)] -pub struct SharedValueSeries { - pub buffer: Rc>>, -} - -impl Debug for SharedValueSeries { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "SharedValueSeries[len: {}]", self.buffer.borrow().len()) - } -} - -impl Object for SharedValueSeries { - fn type_name(&self) -> &'static str { - "SharedValueSeries" - } - fn as_any(&self) -> &dyn Any { - self - } - fn as_series(&self) -> Option<&dyn crate::ast::types::Series> { - Some(self) - } -} - -impl crate::ast::types::Series for SharedValueSeries { - fn get_item(&self, index: usize) -> Option { - self.buffer.borrow().get(index).cloned() - } -} - -/// A read-only series for Records that shares its buffers with the pipeline. -#[derive(Clone)] -pub struct SharedRecordSeries { - pub layout: Arc, - /// Each field shares an Rc> with the Pipe that produces it. - pub field_buffers: Vec>>, -} - -impl Debug for SharedRecordSeries { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "SharedRecordSeries[fields: {}]", self.field_buffers.len()) - } -} - -impl Object for SharedRecordSeries { - fn type_name(&self) -> &'static str { - "SharedRecordSeries" - } - fn as_any(&self) -> &dyn Any { - self - } - fn as_series(&self) -> Option<&dyn crate::ast::types::Series> { - Some(self) - } -} - -impl crate::ast::types::Series for SharedRecordSeries { - fn get_item(&self, index: usize) -> Option { - if self.field_buffers.is_empty() { - return None; - } - - let mut vals = Vec::with_capacity(self.field_buffers.len()); - for buffer in &self.field_buffers { - match buffer.borrow().get_value(index) { - Some(v) => vals.push(v), - None => return None, - } - } - - Some(Value::Record(self.layout.clone(), Rc::new(vals))) - } -} - -// ============================================================================ -// 5. Script Integration (RTL Registration) -// ============================================================================ - -use crate::ast::environment::Environment; -use crate::ast::types::{Purity, Signature}; - -pub fn register(env: &Environment) { - // (create-series layout_record) -> RecordSeries - // Extracts the layout from a sample record and creates an empty SoA RecordSeries. - env.register_native_fn( - "create-series", - StaticType::Function(Box::new(Signature { - params: StaticType::Tuple(vec![StaticType::Any]), // In reality: Record - ret: StaticType::Any, // In reality: Object(RecordSeries) - })), - Purity::Impure, - |args: std::vec::Vec| { - if args.len() != 1 { - panic!("create-series expects exactly 1 argument (a record)"); - } - if let Value::Record(layout, _) = &args[0] { - let series = RecordSeries::new(layout.clone()); - Value::Object(Rc::new(series)) - } else { - panic!("create-series expects a record to extract the layout from") - } - }, - ); - - // (push-series series record) -> Void - // Pushes the values of a record into the parallel arrays of the RecordSeries. - env.register_native_fn( - "push-series", - StaticType::Function(Box::new(Signature { - params: StaticType::Tuple(vec![StaticType::Any, StaticType::Any]), - ret: StaticType::Void, - })), - Purity::Impure, - |args: std::vec::Vec| { - if args.len() != 2 { - panic!("push-series expects exactly 2 arguments (series, record)"); - } - let (series_val, record_val) = (&args[0], &args[1]); - - let record_series = if let Value::Object(obj) = series_val { - obj.as_any().downcast_ref::() - } else { - None - }; - - if let Some(record_series) = record_series { - if let Value::Record(_, values) = record_val { - // Push each value into its corresponding column (SoA push) - for (i, field_val) in values.iter().enumerate() { - if let Some(field_series) = record_series.fields.get(i) { - field_series.borrow_mut().push_value(field_val.clone(), None); - } - } - return Value::Void; - } else { - panic!("push-series expects a record as the second argument"); - } - } - panic!("push-series expects a RecordSeries as the first argument") - }, - ); -} +use std::any::Any; +use std::collections::VecDeque; +use std::fmt::{self, Debug}; +use std::rc::Rc; +use std::sync::Arc; + +use crate::ast::types::{Keyword, Object, RecordLayout, StaticType, Value}; + +// ============================================================================ +// 1. Core Storage Engine +// ============================================================================ + +/// A generic, efficient ring buffer for time series data. +/// (Replaces Delphi's TChunkArray for now with VecDeque for O(1) push/pop) +#[derive(Clone)] +pub struct RingBuffer { + buffer: VecDeque, + total_count: u64, +} + +impl Default for RingBuffer { + fn default() -> Self { + Self::new() + } +} + +impl RingBuffer { + pub fn new() -> Self { + Self { + buffer: VecDeque::new(), + total_count: 0, + } + } + + /// Adds an element. Removes the oldest if the lookback limit is reached. + #[inline] + pub fn push(&mut self, item: T, lookback: Option) { + self.buffer.push_back(item); + self.total_count += 1; + + if let Some(limit) = lookback { + while self.buffer.len() > limit { + self.buffer.pop_front(); + } + } + } + + /// Access in "Financial Style": Index 0 is the newest element. + #[inline] + pub fn get(&self, index: usize) -> Option<&T> { + let len = self.buffer.len(); + if index < len { + self.buffer.get(len - 1 - index) + } else { + None + } + } + + #[inline] + pub fn total_count(&self) -> u64 { + self.total_count + } + + #[inline] + pub fn len(&self) -> usize { + self.buffer.len() + } + + pub fn is_empty(&self) -> bool { + self.buffer.is_empty() + } +} + +// ============================================================================ +// 2. Trait & Primitives for Fast-Paths +// ============================================================================ + +/// Marker trait ensuring our fast arrays only store true, flat scalars +/// (No pointers/heaps like String or Record). +pub trait ScalarValue: Copy + Debug + 'static {} + +impl ScalarValue for f64 {} +impl ScalarValue for i64 {} +impl ScalarValue for bool {} + +/// A highly optimized, type-safe series for primitive values (Float, Int, Bool). +#[derive(Clone)] +pub struct ScalarSeries { + pub data: RingBuffer, + pub type_name: &'static str, +} + +impl ScalarSeries { + pub fn new(type_name: &'static str) -> Self { + Self { + data: RingBuffer::new(), + type_name, + } + } +} + +impl Debug for ScalarSeries { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "ScalarSeries<{}>[len: {}]", + self.type_name, + self.data.len() + ) + } +} + +// Makes the series usable as a dynamic Object for the VM. +impl Object for ScalarSeries { + fn type_name(&self) -> &'static str { + self.type_name + } + fn as_any(&self) -> &dyn Any { + self + } +} + +// ============================================================================ +// 2b. Fallback Series for non-scalar types +// ============================================================================ + +/// A generic series for all non-scalar types (Strings, Tuples, etc.) or mixed data. +#[derive(Clone)] +pub struct ValueSeries { + pub data: RingBuffer, +} + +impl Default for ValueSeries { + fn default() -> Self { + Self::new() + } +} + +impl ValueSeries { + pub fn new() -> Self { + Self { + data: RingBuffer::new(), + } + } +} + +impl Debug for ValueSeries { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "ValueSeries[len: {}]", self.data.len()) + } +} + +impl Object for ValueSeries { + fn type_name(&self) -> &'static str { + "ValueSeries" + } + fn as_any(&self) -> &dyn Any { + self + } +} + +// ============================================================================ +// 3. RecordSeries (SoA - Struct of Arrays) +// ============================================================================ + +/// An interface allowing iteration over fields of a RecordSeries independently +/// of the exact type (Type Erasure for member arrays). +pub trait SeriesMember: Object { + /// Gets the value at the specified lookback index as a dynamic Value. + fn get_value(&self, index: usize) -> Option; + + /// Pushes a dynamic Value into the series, performing the necessary downcast. + fn push_value(&mut self, value: Value, limit: Option); +} + +impl SeriesMember for ScalarSeries { + fn get_value(&self, index: usize) -> Option { + self.data.get(index).map(|&v| Value::Float(v)) + } + fn push_value(&mut self, value: Value, limit: Option) { + if let Value::Float(v) = value { + self.data.push(v, limit); + } + } +} + +impl SeriesMember for ScalarSeries { + fn get_value(&self, index: usize) -> Option { + self.data.get(index).map(|&v| Value::Int(v)) + } + fn push_value(&mut self, value: Value, limit: Option) { + if let Value::Int(v) = value { + self.data.push(v, limit); + } else if let Value::DateTime(v) = value { + self.data.push(v, limit); + } + } +} + +impl SeriesMember for ScalarSeries { + fn get_value(&self, index: usize) -> Option { + self.data.get(index).map(|&v| Value::Bool(v)) + } + fn push_value(&mut self, value: Value, limit: Option) { + if let Value::Bool(v) = value { + self.data.push(v, limit); + } + } +} + +impl SeriesMember for ValueSeries { + fn get_value(&self, index: usize) -> Option { + self.data.get(index).cloned() + } + fn push_value(&mut self, value: Value, limit: Option) { + self.data.push(value, limit); + } +} + +/// The "Struct of Arrays" implementation for Records (e.g., Ticks). +/// Instead of holding a large array of Records (AoS), this series +/// splits the record and stores each field in a separate (parallel) series. +#[derive(Clone)] +pub struct RecordSeries { + layout: Arc, + /// Each field of the record gets its own series (SoA). + /// We use Rc so we can iterate fields individually, + /// but also pass them as 0-Copy references to indicators! + fields: Vec>>, +} + +impl RecordSeries { + /// Creates a new RecordSeries matching the given layout. + pub fn new(layout: Arc) -> Self { + let mut fields: Vec>> = + Vec::with_capacity(layout.fields.len()); + + for (_, static_type) in &layout.fields { + // Automatically select the optimal storage representation based on the field's static type. + let member: Rc> = match static_type { + StaticType::Float => Rc::new(std::cell::RefCell::new(ScalarSeries::::new( + "FloatSeries", + ))), + StaticType::Int | StaticType::DateTime => Rc::new(std::cell::RefCell::new( + ScalarSeries::::new("IntSeries"), + )), + StaticType::Bool => Rc::new(std::cell::RefCell::new(ScalarSeries::::new( + "BoolSeries", + ))), + // Fallback for everything else (Text, Lists, nested Records without SoA, etc.) + _ => Rc::new(std::cell::RefCell::new(ValueSeries::new())), + }; + fields.push(member); + } + + Self { layout, fields } + } + + /// The magical 0-Copy Field Mapper! + /// Returns the complete series of a single field. + /// Example: `my_ticks.field(Keyword::intern("close"))` + /// returns an `Rc>>`. No copy involved! + pub fn field(&self, key: Keyword) -> Option>> { + self.layout + .index_of(key) + .map(|idx| self.fields[idx].clone()) + } + + /// Dynamically reconstructs the full Record at the given lookback index. + /// This is an O(N) operation where N is the number of fields, meaning it breaks + /// the 0-Copy performance path, but it's essential for idiomatic AST access `(series 0)`. + pub fn get_record(&self, index: usize) -> Option { + if self.fields.is_empty() { + return None; + } + + let mut vals = Vec::with_capacity(self.fields.len()); + for field in &self.fields { + match field.borrow().get_value(index) { + Some(v) => vals.push(v), + None => return None, // Out of bounds or incomplete + } + } + + Some(Value::Record(self.layout.clone(), Rc::new(vals))) + } +} + +impl Debug for RecordSeries { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "RecordSeries[fields: {}]", self.fields.len()) + } +} + +impl Object for RecordSeries { + fn type_name(&self) -> &'static str { + "RecordSeries" + } + fn as_any(&self) -> &dyn Any { + self + } + fn as_series(&self) -> Option<&dyn crate::ast::types::Series> { + Some(self) + } +} + +impl crate::ast::types::Series for RecordSeries { + fn get_item(&self, index: usize) -> Option { + self.get_record(index) + } +} + +// ============================================================================ +// 4. Series View (Safe, 0-Copy access bridge for the VM) +// ============================================================================ + +/// A lightweight, 0-copy wrapper around a specific column (field) of a RecordSeries. +/// +/// Why is this needed? +/// In a SoA (Struct of Arrays) architecture, columns like `open` or `close` +/// are stored as `Rc>` inside the `RecordSeries`. +/// The `RefCell` is crucial for runtime mutability (so the engine can push new ticks). +/// However, the VM's `Value::Object` expects a pure `Rc`. +/// +/// `SeriesView` bridges this gap safely: It holds the shared reference and implements `Object` itself, +/// acting as a fast, read-only "window" into the underlying column data without copying any arrays. +#[derive(Clone)] +pub struct SeriesView { + pub inner: Rc>, + pub field_name: Keyword, +} + +impl SeriesView { + pub fn new(inner: Rc>, field_name: Keyword) -> Self { + Self { inner, field_name } + } +} + +impl Debug for SeriesView { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "SeriesView[field: {}]", self.field_name.name()) + } +} + +impl Object for SeriesView { + fn type_name(&self) -> &'static str { + "SeriesView" + } + + // This allows the VM or specialized extractors to downcast the View back to its concrete type if needed. + fn as_any(&self) -> &dyn Any { + self + } + + fn as_series(&self) -> Option<&dyn crate::ast::types::Series> { + Some(self) + } +} + +impl crate::ast::types::Series for SeriesView { + fn get_item(&self, index: usize) -> Option { + self.inner.borrow().get_value(index) + } +} + +// ============================================================================ +// 4b. Shared Series (Read-Only Reactive Storage) +// ============================================================================ + +/// A read-only series that shares its underlying buffer with the pipeline engine. +/// This is the basis for the "SharedSeries" in the Dual Series Architecture. +#[derive(Clone)] +pub struct SharedSeries { + pub buffer: Rc>>, + pub type_name: &'static str, + pub converter: fn(T) -> Value, +} + +impl Debug for SharedSeries { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "SharedSeries<{}>[len: {}]", + self.type_name, + self.buffer.borrow().len() + ) + } +} + +impl Object for SharedSeries { + fn type_name(&self) -> &'static str { + self.type_name + } + fn as_any(&self) -> &dyn Any { + self + } + fn as_series(&self) -> Option<&dyn crate::ast::types::Series> { + Some(self) + } +} + +impl crate::ast::types::Series for SharedSeries { + fn get_item(&self, index: usize) -> Option { + self.buffer + .borrow() + .get(index) + .map(|&v| (self.converter)(v)) + } +} + +/// A read-only series for generic Values that shares its buffer with the pipeline. +#[derive(Clone)] +pub struct SharedValueSeries { + pub buffer: Rc>>, +} + +impl Debug for SharedValueSeries { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "SharedValueSeries[len: {}]", self.buffer.borrow().len()) + } +} + +impl Object for SharedValueSeries { + fn type_name(&self) -> &'static str { + "SharedValueSeries" + } + fn as_any(&self) -> &dyn Any { + self + } + fn as_series(&self) -> Option<&dyn crate::ast::types::Series> { + Some(self) + } +} + +impl crate::ast::types::Series for SharedValueSeries { + fn get_item(&self, index: usize) -> Option { + self.buffer.borrow().get(index).cloned() + } +} + +/// A read-only series for Records that shares its buffers with the pipeline. +#[derive(Clone)] +pub struct SharedRecordSeries { + pub layout: Arc, + /// Each field shares an Rc> with the Pipe that produces it. + pub field_buffers: Vec>>, +} + +impl Debug for SharedRecordSeries { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "SharedRecordSeries[fields: {}]", + self.field_buffers.len() + ) + } +} + +impl Object for SharedRecordSeries { + fn type_name(&self) -> &'static str { + "SharedRecordSeries" + } + fn as_any(&self) -> &dyn Any { + self + } + fn as_series(&self) -> Option<&dyn crate::ast::types::Series> { + Some(self) + } +} + +impl crate::ast::types::Series for SharedRecordSeries { + fn get_item(&self, index: usize) -> Option { + if self.field_buffers.is_empty() { + return None; + } + + let mut vals = Vec::with_capacity(self.field_buffers.len()); + for buffer in &self.field_buffers { + match buffer.borrow().get_value(index) { + Some(v) => vals.push(v), + None => return None, + } + } + + Some(Value::Record(self.layout.clone(), Rc::new(vals))) + } +} + +// ============================================================================ +// 5. Script Integration (RTL Registration) +// ============================================================================ + +use crate::ast::environment::Environment; +use crate::ast::types::{Purity, Signature}; + +pub fn register(env: &Environment) { + // (create-series layout_record) -> RecordSeries + // Extracts the layout from a sample record and creates an empty SoA RecordSeries. + env.register_native_fn( + "create-series", + StaticType::Function(Box::new(Signature { + params: StaticType::Tuple(vec![StaticType::Any]), // In reality: Record + ret: StaticType::Any, // In reality: Object(RecordSeries) + })), + Purity::Impure, + |args: std::vec::Vec| { + if args.len() != 1 { + panic!("create-series expects exactly 1 argument (a record)"); + } + if let Value::Record(layout, _) = &args[0] { + let series = RecordSeries::new(layout.clone()); + Value::Object(Rc::new(series)) + } else { + panic!("create-series expects a record to extract the layout from") + } + }, + ); + + // (push-series series record) -> Void + // Pushes the values of a record into the parallel arrays of the RecordSeries. + env.register_native_fn( + "push-series", + StaticType::Function(Box::new(Signature { + params: StaticType::Tuple(vec![StaticType::Any, StaticType::Any]), + ret: StaticType::Void, + })), + Purity::Impure, + |args: std::vec::Vec| { + if args.len() != 2 { + panic!("push-series expects exactly 2 arguments (series, record)"); + } + let (series_val, record_val) = (&args[0], &args[1]); + + let record_series = if let Value::Object(obj) = series_val { + obj.as_any().downcast_ref::() + } else { + None + }; + + if let Some(record_series) = record_series { + if let Value::Record(_, values) = record_val { + // Push each value into its corresponding column (SoA push) + for (i, field_val) in values.iter().enumerate() { + if let Some(field_series) = record_series.fields.get(i) { + field_series + .borrow_mut() + .push_value(field_val.clone(), None); + } + } + return Value::Void; + } else { + panic!("push-series expects a record as the second argument"); + } + } + panic!("push-series expects a RecordSeries as the first argument") + }, + ); +} diff --git a/src/ast/rtl/streams.rs b/src/ast/rtl/streams.rs index 095732a..9781eed 100644 --- a/src/ast/rtl/streams.rs +++ b/src/ast/rtl/streams.rs @@ -1,593 +1,641 @@ -use std::rc::Rc; -use std::cell::RefCell; -use crate::ast::types::Value; - -/// A Signal is the "packet" flowing through the reactive pipeline. -/// It represents a value produced at a specific logical time (cycle_id). -#[derive(Debug, Clone)] -pub struct Signal { - pub cycle_id: u64, - pub value: Value, -} - -/// A Stream is a stateless provider of signals. -/// It doesn't "own" the data, it just knows how to get the current one. -pub trait Stream { - fn current_signal(&self) -> Option; -} - -/// An Observer is a node in the pipeline that reacts to new signals. -/// (e.g., a Pipe or a SharedSeries buffer). -pub trait Observer { - /// Notifies the observer about a new signal in the current cycle. - /// `source_index` identifies which input stream provided the value. - fn notify(&mut self, source_index: usize, cycle_id: u64, value: Value); -} - -/// A lightweight adapter to map an unknown source index to a specific target index. -/// This prevents index collisions when a Pipe listens to multiple independent RootStreams. -pub struct SourceAdapter { - pub target: Rc>, - pub target_index: usize, -} - -impl Observer for SourceAdapter { - fn notify(&mut self, _ignored_source: usize, cycle_id: u64, value: Value) { - self.target.borrow_mut().notify(self.target_index, cycle_id, value); - } -} - -/// Polymorphic Interface for any stream that can accept observers (like Delphi's IStream). -pub trait ObservableStream { - fn add_observer(&self, observer: Rc>); -} - -impl ObservableStream for std::cell::RefCell { - fn add_observer(&self, observer: Rc>) { - self.borrow().add_observer(observer); - } -} - -/// A generic wrapper to pass ANY ObservableStream (Root, Pipe, etc.) as an Object to the VM. -#[derive(Clone)] -pub struct StreamNode { - pub inner: Rc, -} - -impl std::fmt::Debug for StreamNode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "StreamNode") - } -} - -impl crate::ast::types::Object for StreamNode { - fn type_name(&self) -> &'static str { - "StreamNode" - } - fn as_any(&self) -> &dyn std::any::Any { - self - } -} - -/// A node that represents the result of a `pipe` statement. -/// It acts as a Stream (to connect downstream pipes) AND as a Series (for script lookbacks). -#[derive(Clone)] -pub struct PipelineNode { - pub stream: Rc, - pub series: Rc, -} - -impl std::fmt::Debug for PipelineNode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "PipelineNode") - } -} - -impl crate::ast::types::Object for PipelineNode { - fn type_name(&self) -> &'static str { - "PipelineNode" - } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn as_series(&self) -> Option<&dyn crate::ast::types::Series> { - Some(self.series.as_ref()) - } -} - -/// The RootStream is the "Clock" and data source of the entire pipeline. -/// It generates the monotonic `cycle_id` and triggers the observers. -pub struct RootStream { - current_cycle: std::cell::Cell, - observers: RefCell>>>, -} - -impl ObservableStream for RootStream { - fn add_observer(&self, observer: Rc>) { - self.observers.borrow_mut().push(observer); - } -} - -impl Default for RootStream { - fn default() -> Self { - Self::new() - } -} - -impl RootStream { - pub fn new() -> Self { - Self { - current_cycle: std::cell::Cell::new(0), - observers: RefCell::new(Vec::new()), - } - } - - /// Advances the pipeline to the next cycle and propagates a value. - pub fn tick(&self, value: Value) { - let next_cycle = self.current_cycle.get() + 1; - self.current_cycle.set(next_cycle); - - // Propagate to all observers. - // We use a local borrow of the observers list to keep the cell borrow short. - let obs_list = self.observers.borrow(); - for obs in obs_list.iter() { - // Root observers are always at source_index 0. - obs.borrow_mut().notify(0, next_cycle, value.clone()); - } - } - - pub fn current_cycle(&self) -> u64 { - self.current_cycle.get() - } - - pub fn add_observer(&self, observer: Rc>) { - self.observers.borrow_mut().push(observer); - } -} - -/// A PipeStream is a reactive node that transforms inputs via a lambda. -/// It implements "Barrier Synchronization": It only executes when all inputs -/// have reported a value for the same cycle_id. -pub struct PipeStream { - pub name: String, - /// The inputs this pipe is observing. - /// In a real system, these would be other Streams. - /// For the MVP, we assume the Pipe is notified by the Root or its parents. - pub input_count: usize, - /// Tracks the last cycle_id received from each input. - last_cycle_per_input: Vec, - /// Stores the current value for each input to construct the argument tuple. - current_values: Vec, - /// The current output signal of this pipe. - current_signal: RefCell>, - /// The executable closure representing the Lambda. Expects a Vec of arguments. - pub executor: Option) -> Value>>, - /// Observers of THIS pipe. - observers: RefCell>>>, -} - -impl PipeStream { - pub fn new( - name: String, - input_count: usize, - executor: Option) -> Value>> - ) -> Self { - Self { - name, - input_count, - last_cycle_per_input: vec![0; input_count], - current_values: vec![Value::Void; input_count], - current_signal: RefCell::new(None), - executor, - observers: RefCell::new(Vec::new()), - } - } -} - -impl ObservableStream for PipeStream { - fn add_observer(&self, observer: Rc>) { - self.observers.borrow_mut().push(observer); - } -} - -impl Stream for PipeStream { - fn current_signal(&self) -> Option { - self.current_signal.borrow().clone() - } -} - -impl Observer for PipeStream { - fn notify(&mut self, source_index: usize, cycle_id: u64, value: Value) { - let barrier_reached = { - if source_index < self.input_count { - self.last_cycle_per_input[source_index] = cycle_id; - self.current_values[source_index] = value; - } - // Check if all inputs reached the same cycle. - self.last_cycle_per_input.iter().all(|&c| c == cycle_id) - }; - - if barrier_reached { - // 1. Prepare Arguments for Lambda (Current values of all inputs) - let args = self.current_values.clone(); - - // 2. Execute Lambda using the encapsulated VM executor - let result = if let Some(exec) = &mut self.executor { - exec(args) - } else { - self.current_values[0].clone() // Identity bypass (defaults to first input) - }; - - // 3. Handle Void case! (Filter pattern) - if matches!(result, Value::Void) { - return; // Act as a filter: do not emit, do not push. - } - - // 4. Update Current Signal - let new_signal = Signal { cycle_id, value: result }; - *self.current_signal.borrow_mut() = Some(new_signal.clone()); - - // 5. Notify Observers (Always at source_index 0 of the NEXT pipe) - let obs_list = self.observers.borrow(); - for obs in obs_list.iter() { - obs.borrow_mut().notify(0, cycle_id, new_signal.value.clone()); - } - } - } -} - -/// A specialized observer that pushes incoming signals into a SharedSeries buffer. -pub struct SeriesPusher { - pub buffer: Rc>>, - pub lookback: Option, - pub extractor: fn(Value) -> Option, -} - -impl Observer for SeriesPusher { - fn notify(&mut self, _source_index: usize, _cycle_id: u64, value: Value) { - if let Some(v) = (self.extractor)(value) { - self.buffer.borrow_mut().push(v, self.lookback); - } - } -} - -/// A specialized observer that pushes incoming signals into a generic SharedValueSeries buffer. -pub struct ValuePusher { - pub buffer: Rc>>, - pub lookback: Option, -} - -impl Observer for ValuePusher { - fn notify(&mut self, _source_index: usize, _cycle_id: u64, value: Value) { - self.buffer.borrow_mut().push(value, self.lookback); - } -} - -/// A specialized observer that splits a Record into its fields and pushes them into SoA buffers. -pub struct RecordPusher { - pub field_buffers: Vec>>, - pub lookback: Option, -} - -impl Observer for RecordPusher { - fn notify(&mut self, _source_index: usize, _cycle_id: u64, value: Value) { - if let Value::Record(_, values) = value { - for (i, v) in values.iter().enumerate() { - if let Some(buf) = self.field_buffers.get(i) { - buf.borrow_mut().push_value(v.clone(), self.lookback); - } - } - } - } -} - -/// Factory function to build a specialized pipeline node based on the output type. -/// This keeps the VM "dumb" and moves the buffer selection logic to the RTL. -pub fn build_pipeline_node( - inputs: Vec>, - executor: Box) -> Value>, - out_type: &StaticType, -) -> Rc { - use crate::ast::rtl::series::*; - - let pipe = Rc::new(RefCell::new(PipeStream::new( - "pipe".to_string(), - inputs.len(), - Some(executor), - ))); - - // Connect inputs to the pipe - for (i, input) in inputs.into_iter().enumerate() { - let adapter = Rc::new(RefCell::new(SourceAdapter { - target: pipe.clone(), - target_index: i, - })); - input.add_observer(adapter); - } - - let series: Rc = match out_type { - StaticType::Float => { - let buffer = Rc::new(RefCell::new(RingBuffer::::new())); - let pusher = Rc::new(RefCell::new(SeriesPusher { - buffer: buffer.clone(), - lookback: Some(100), // TODO: Configuration - extractor: |v| if let Value::Float(f) = v { Some(f) } else { None }, - })); - pipe.borrow().add_observer(pusher); - Rc::new(SharedSeries { - buffer, - type_name: "SharedFloatSeries", - converter: |v| Value::Float(v), - }) - } - StaticType::Int | StaticType::DateTime => { - let buffer = Rc::new(RefCell::new(RingBuffer::::new())); - let pusher = Rc::new(RefCell::new(SeriesPusher { - buffer: buffer.clone(), - lookback: Some(100), - extractor: |v| match v { - Value::Int(i) => Some(i), - Value::DateTime(i) => Some(i), - _ => None, - }, - })); - pipe.borrow().add_observer(pusher); - Rc::new(SharedSeries { - buffer, - type_name: "SharedIntSeries", - converter: |v| Value::Int(v), - }) - } - StaticType::Bool => { - let buffer = Rc::new(RefCell::new(RingBuffer::::new())); - let pusher = Rc::new(RefCell::new(SeriesPusher { - buffer: buffer.clone(), - lookback: Some(100), - extractor: |v| if let Value::Bool(b) = v { Some(b) } else { None }, - })); - pipe.borrow().add_observer(pusher); - Rc::new(SharedSeries { - buffer, - type_name: "SharedBoolSeries", - converter: |v| Value::Bool(v), - }) - } - StaticType::Record(layout) => { - let mut field_buffers = Vec::with_capacity(layout.fields.len()); - for (_, ty) in &layout.fields { - let member: Rc> = match ty { - StaticType::Float => Rc::new(RefCell::new(ScalarSeries::::new("FloatSeries"))), - StaticType::Int | StaticType::DateTime => { - Rc::new(RefCell::new(ScalarSeries::::new("IntSeries"))) - } - StaticType::Bool => Rc::new(RefCell::new(ScalarSeries::::new("BoolSeries"))), - _ => Rc::new(RefCell::new(ValueSeries::new())), - }; - field_buffers.push(member); - } - - let pusher = Rc::new(RefCell::new(RecordPusher { - field_buffers: field_buffers.clone(), - lookback: Some(100), - })); - pipe.borrow().add_observer(pusher); - - Rc::new(SharedRecordSeries { - layout: layout.clone(), - field_buffers, - }) - } - _ => { - let buffer = Rc::new(RefCell::new(RingBuffer::::new())); - let pusher = Rc::new(RefCell::new(ValuePusher { - buffer: buffer.clone(), - lookback: Some(100), - })); - pipe.borrow().add_observer(pusher); - Rc::new(SharedValueSeries { buffer }) - } - }; - - Rc::new(PipelineNode { - stream: pipe, - series, - }) -} - -// ============================================================================ -// Script Integration (RTL Registration) -// ============================================================================ - -use crate::ast::environment::Environment; -use crate::ast::types::{Purity, Signature, StaticType, Keyword, RecordLayout}; - -pub fn register(env: &Environment) { - // (create-random-ohlc seed limit) -> StreamNode - let generators = env.pipeline_generators.clone(); - - // Define the OHLC layout for typing - let ohlc_layout = RecordLayout::get_or_create(vec![ - (Keyword::intern("open"), StaticType::Float), - (Keyword::intern("high"), StaticType::Float), - (Keyword::intern("low"), StaticType::Float), - (Keyword::intern("close"), StaticType::Float), - ]); - - env.register_native_fn( - "create-random-ohlc", - StaticType::Function(Box::new(Signature { - params: StaticType::Tuple(vec![StaticType::Int, StaticType::Int]), - ret: StaticType::Series(Box::new(StaticType::Record(ohlc_layout.clone()))), - })), - Purity::Impure, // Modifies global generator registry - move |args: std::vec::Vec| { - if args.len() != 2 { - panic!("create-random-ohlc expects exactly 2 arguments (seed, limit)"); - } - let seed = if let Value::Int(s) = args[0] { s as u64 } else { 0 }; - let limit = if let Value::Int(l) = args[1] { l as usize } else { 0 }; - - // 1. Create the RootStream - let root_stream = Rc::new(RootStream::new()); - let stream_node = StreamNode { inner: root_stream.clone() }; - - // 2. Setup the Layout for OHLC records - let layout = RecordLayout::get_or_create(vec![ - (Keyword::intern("open"), StaticType::Float), - (Keyword::intern("high"), StaticType::Float), - (Keyword::intern("low"), StaticType::Float), - (Keyword::intern("close"), StaticType::Float), - ]); - - // 3. Create the stateful generator closure - let mut current_tick = 0; - let mut last_close = 100.0; - - // We use a local PRNG instance for reproducibility based on the seed - let mut rng = fastrand::Rng::with_seed(seed); - - let generator = move || -> bool { - if current_tick >= limit { - return false; // Exhausted - } - - // Generate random OHLC (Random Walk) - let change = (rng.f64() - 0.5) * 2.0; - let open = last_close; - let high = open + (rng.f64() * 2.0).abs(); - let low = open - (rng.f64() * 2.0).abs(); - let close = open + change; - last_close = close; - - let record = Value::Record( - layout.clone(), - Rc::new(vec![ - Value::Float(open), - Value::Float(high), - Value::Float(low), - Value::Float(close), - ]) - ); - - // Pump the signal into the RootStream - root_stream.tick(record); - - current_tick += 1; - true // Still active - }; - - // 4. Register the generator in the Environment - generators.borrow_mut().push(Box::new(generator)); - - // 5. Return the stream reference to the script - Value::Object(Rc::new(stream_node)) - }, - ); - - // (create-ticker condition-closure) -> StreamNode - let ticker_generators = env.pipeline_generators.clone(); - let globals = env.global_values.clone(); - - env.register_native_fn( - "create-ticker", - StaticType::Function(Box::new(Signature { - params: StaticType::Tuple(vec![StaticType::Any]), // Expects a closure - ret: StaticType::Any, // Returns StreamNode - })), - Purity::Impure, - move |args: std::vec::Vec| { - if args.len() != 1 { - panic!("create-ticker expects exactly 1 argument (the condition closure)"); - } - - let closure_obj = if let Value::Object(obj) = &args[0] { - obj.clone() - } else { - panic!("create-ticker expects a closure as its argument"); - }; - - // 1. Create the RootStream - let root_stream = Rc::new(RootStream::new()); - let stream_node = StreamNode { inner: root_stream.clone() }; - - // 2. Setup isolated VM - let mut ticker_vm = crate::ast::vm::VM::new(globals.clone()); - let my_closure = closure_obj.clone(); - - // 3. Create generator - let generator = move || -> bool { - match ticker_vm.run_with_args(my_closure.clone(), vec![]) { - Ok(Value::Bool(b)) => { - if b { - // Ticker pulses with a simple `true` value or `Void` - // We use true here so the pipe receives something tangible. - root_stream.tick(Value::Bool(true)); - true - } else { - false // Exhausted - } - } - Ok(_) => panic!("create-ticker closure must return a boolean"), - Err(e) => panic!("create-ticker closure execution failed: {}", e), - } - }; - - // 4. Register the generator - ticker_generators.borrow_mut().push(Box::new(generator)); - - // 5. Return stream - Value::Object(Rc::new(stream_node)) - } - ); -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ast::types::Value; - - #[test] - fn test_root_to_pipe_flow() { - let root = RootStream::new(); - let pipe = Rc::new(RefCell::new(PipeStream::new("test-pipe".to_string(), 1, None))); - - root.add_observer(pipe.clone()); - - // Cycle 1: Root ticks 10.0 - root.tick(Value::Float(10.0)); - - let sig = pipe.borrow().current_signal().unwrap(); - assert_eq!(sig.cycle_id, 1); - if let Value::Float(v) = sig.value { - assert_eq!(v, 10.0); - } else { - panic!("Value must be Float(10.0)"); - } - - // Cycle 2: Root ticks 20.0 - root.tick(Value::Float(20.0)); - let sig2 = pipe.borrow().current_signal().unwrap(); - assert_eq!(sig2.cycle_id, 2); - if let Value::Float(v) = sig2.value { - assert_eq!(v, 20.0); - } else { - panic!("Value must be Float(20.0)"); - } - } - - #[test] - fn test_barrier_sync() { - // Pipe with 2 inputs - let pipe = Rc::new(RefCell::new(PipeStream::new("barrier-pipe".to_string(), 2, None))); - - // Manual notifications simulate different input streams - pipe.borrow_mut().notify(0, 1, Value::Float(10.0)); - assert!(pipe.borrow().current_signal().is_none(), "Barrier should NOT be reached after 1st input"); - - pipe.borrow_mut().notify(1, 1, Value::Float(20.0)); - assert!(pipe.borrow().current_signal().is_some(), "Barrier SHOULD be reached after 2nd input"); - - let sig = pipe.borrow().current_signal().unwrap(); - assert_eq!(sig.cycle_id, 1); - } -} +use crate::ast::types::Value; +use std::cell::RefCell; +use std::rc::Rc; + +/// A Signal is the "packet" flowing through the reactive pipeline. +/// It represents a value produced at a specific logical time (cycle_id). +#[derive(Debug, Clone)] +pub struct Signal { + pub cycle_id: u64, + pub value: Value, +} + +/// A Stream is a stateless provider of signals. +/// It doesn't "own" the data, it just knows how to get the current one. +pub trait Stream { + fn current_signal(&self) -> Option; +} + +/// An Observer is a node in the pipeline that reacts to new signals. +/// (e.g., a Pipe or a SharedSeries buffer). +pub trait Observer { + /// Notifies the observer about a new signal in the current cycle. + /// `source_index` identifies which input stream provided the value. + fn notify(&mut self, source_index: usize, cycle_id: u64, value: Value); +} + +/// A lightweight adapter to map an unknown source index to a specific target index. +/// This prevents index collisions when a Pipe listens to multiple independent RootStreams. +pub struct SourceAdapter { + pub target: Rc>, + pub target_index: usize, +} + +impl Observer for SourceAdapter { + fn notify(&mut self, _ignored_source: usize, cycle_id: u64, value: Value) { + self.target + .borrow_mut() + .notify(self.target_index, cycle_id, value); + } +} + +/// Polymorphic Interface for any stream that can accept observers (like Delphi's IStream). +pub trait ObservableStream { + fn add_observer(&self, observer: Rc>); +} + +impl ObservableStream for std::cell::RefCell { + fn add_observer(&self, observer: Rc>) { + self.borrow().add_observer(observer); + } +} + +/// A generic wrapper to pass ANY ObservableStream (Root, Pipe, etc.) as an Object to the VM. +#[derive(Clone)] +pub struct StreamNode { + pub inner: Rc, +} + +impl std::fmt::Debug for StreamNode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "StreamNode") + } +} + +impl crate::ast::types::Object for StreamNode { + fn type_name(&self) -> &'static str { + "StreamNode" + } + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +/// A node that represents the result of a `pipe` statement. +/// It acts as a Stream (to connect downstream pipes) AND as a Series (for script lookbacks). +#[derive(Clone)] +pub struct PipelineNode { + pub stream: Rc, + pub series: Rc, +} + +impl std::fmt::Debug for PipelineNode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "PipelineNode") + } +} + +impl crate::ast::types::Object for PipelineNode { + fn type_name(&self) -> &'static str { + "PipelineNode" + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn as_series(&self) -> Option<&dyn crate::ast::types::Series> { + Some(self.series.as_ref()) + } +} + +/// The RootStream is the "Clock" and data source of the entire pipeline. +/// It generates the monotonic `cycle_id` and triggers the observers. +pub struct RootStream { + current_cycle: std::cell::Cell, + observers: RefCell>>>, +} + +impl ObservableStream for RootStream { + fn add_observer(&self, observer: Rc>) { + self.observers.borrow_mut().push(observer); + } +} + +impl Default for RootStream { + fn default() -> Self { + Self::new() + } +} + +impl RootStream { + pub fn new() -> Self { + Self { + current_cycle: std::cell::Cell::new(0), + observers: RefCell::new(Vec::new()), + } + } + + /// Advances the pipeline to the next cycle and propagates a value. + pub fn tick(&self, value: Value) { + let next_cycle = self.current_cycle.get() + 1; + self.current_cycle.set(next_cycle); + + // Propagate to all observers. + // We use a local borrow of the observers list to keep the cell borrow short. + let obs_list = self.observers.borrow(); + for obs in obs_list.iter() { + // Root observers are always at source_index 0. + obs.borrow_mut().notify(0, next_cycle, value.clone()); + } + } + + pub fn current_cycle(&self) -> u64 { + self.current_cycle.get() + } + + pub fn add_observer(&self, observer: Rc>) { + self.observers.borrow_mut().push(observer); + } +} + +/// A PipeStream is a reactive node that transforms inputs via a lambda. +/// It implements "Barrier Synchronization": It only executes when all inputs +/// have reported a value for the same cycle_id. +pub struct PipeStream { + pub name: String, + /// The inputs this pipe is observing. + /// In a real system, these would be other Streams. + /// For the MVP, we assume the Pipe is notified by the Root or its parents. + pub input_count: usize, + /// Tracks the last cycle_id received from each input. + last_cycle_per_input: Vec, + /// Stores the current value for each input to construct the argument tuple. + current_values: Vec, + /// The current output signal of this pipe. + current_signal: RefCell>, + /// The executable closure representing the Lambda. Expects a Vec of arguments. + pub executor: Option) -> Value>>, + /// Observers of THIS pipe. + observers: RefCell>>>, +} + +impl PipeStream { + pub fn new( + name: String, + input_count: usize, + executor: Option) -> Value>>, + ) -> Self { + Self { + name, + input_count, + last_cycle_per_input: vec![0; input_count], + current_values: vec![Value::Void; input_count], + current_signal: RefCell::new(None), + executor, + observers: RefCell::new(Vec::new()), + } + } +} + +impl ObservableStream for PipeStream { + fn add_observer(&self, observer: Rc>) { + self.observers.borrow_mut().push(observer); + } +} + +impl Stream for PipeStream { + fn current_signal(&self) -> Option { + self.current_signal.borrow().clone() + } +} + +impl Observer for PipeStream { + fn notify(&mut self, source_index: usize, cycle_id: u64, value: Value) { + let barrier_reached = { + if source_index < self.input_count { + self.last_cycle_per_input[source_index] = cycle_id; + self.current_values[source_index] = value; + } + // Check if all inputs reached the same cycle. + self.last_cycle_per_input.iter().all(|&c| c == cycle_id) + }; + + if barrier_reached { + // 1. Prepare Arguments for Lambda (Current values of all inputs) + let args = self.current_values.clone(); + + // 2. Execute Lambda using the encapsulated VM executor + let result = if let Some(exec) = &mut self.executor { + exec(args) + } else { + self.current_values[0].clone() // Identity bypass (defaults to first input) + }; + + // 3. Handle Void case! (Filter pattern) + if matches!(result, Value::Void) { + return; // Act as a filter: do not emit, do not push. + } + + // 4. Update Current Signal + let new_signal = Signal { + cycle_id, + value: result, + }; + *self.current_signal.borrow_mut() = Some(new_signal.clone()); + + // 5. Notify Observers (Always at source_index 0 of the NEXT pipe) + let obs_list = self.observers.borrow(); + for obs in obs_list.iter() { + obs.borrow_mut() + .notify(0, cycle_id, new_signal.value.clone()); + } + } + } +} + +/// A specialized observer that pushes incoming signals into a SharedSeries buffer. +pub struct SeriesPusher { + pub buffer: Rc>>, + pub lookback: Option, + pub extractor: fn(Value) -> Option, +} + +impl Observer for SeriesPusher { + fn notify(&mut self, _source_index: usize, _cycle_id: u64, value: Value) { + if let Some(v) = (self.extractor)(value) { + self.buffer.borrow_mut().push(v, self.lookback); + } + } +} + +/// A specialized observer that pushes incoming signals into a generic SharedValueSeries buffer. +pub struct ValuePusher { + pub buffer: Rc>>, + pub lookback: Option, +} + +impl Observer for ValuePusher { + fn notify(&mut self, _source_index: usize, _cycle_id: u64, value: Value) { + self.buffer.borrow_mut().push(value, self.lookback); + } +} + +/// A specialized observer that splits a Record into its fields and pushes them into SoA buffers. +pub struct RecordPusher { + pub field_buffers: Vec>>, + pub lookback: Option, +} + +impl Observer for RecordPusher { + fn notify(&mut self, _source_index: usize, _cycle_id: u64, value: Value) { + if let Value::Record(_, values) = value { + for (i, v) in values.iter().enumerate() { + if let Some(buf) = self.field_buffers.get(i) { + buf.borrow_mut().push_value(v.clone(), self.lookback); + } + } + } + } +} + +/// Factory function to build a specialized pipeline node based on the output type. +/// This keeps the VM "dumb" and moves the buffer selection logic to the RTL. +pub fn build_pipeline_node( + inputs: Vec>, + executor: Box) -> Value>, + out_type: &StaticType, +) -> Rc { + use crate::ast::rtl::series::*; + + let pipe = Rc::new(RefCell::new(PipeStream::new( + "pipe".to_string(), + inputs.len(), + Some(executor), + ))); + + // Connect inputs to the pipe + for (i, input) in inputs.into_iter().enumerate() { + let adapter = Rc::new(RefCell::new(SourceAdapter { + target: pipe.clone(), + target_index: i, + })); + input.add_observer(adapter); + } + + let series: Rc = match out_type { + StaticType::Float => { + let buffer = Rc::new(RefCell::new(RingBuffer::::new())); + let pusher = Rc::new(RefCell::new(SeriesPusher { + buffer: buffer.clone(), + lookback: Some(100), // TODO: Configuration + extractor: |v| { + if let Value::Float(f) = v { + Some(f) + } else { + None + } + }, + })); + pipe.borrow().add_observer(pusher); + Rc::new(SharedSeries { + buffer, + type_name: "SharedFloatSeries", + converter: |v| Value::Float(v), + }) + } + StaticType::Int | StaticType::DateTime => { + let buffer = Rc::new(RefCell::new(RingBuffer::::new())); + let pusher = Rc::new(RefCell::new(SeriesPusher { + buffer: buffer.clone(), + lookback: Some(100), + extractor: |v| match v { + Value::Int(i) => Some(i), + Value::DateTime(i) => Some(i), + _ => None, + }, + })); + pipe.borrow().add_observer(pusher); + Rc::new(SharedSeries { + buffer, + type_name: "SharedIntSeries", + converter: |v| Value::Int(v), + }) + } + StaticType::Bool => { + let buffer = Rc::new(RefCell::new(RingBuffer::::new())); + let pusher = Rc::new(RefCell::new(SeriesPusher { + buffer: buffer.clone(), + lookback: Some(100), + extractor: |v| { + if let Value::Bool(b) = v { + Some(b) + } else { + None + } + }, + })); + pipe.borrow().add_observer(pusher); + Rc::new(SharedSeries { + buffer, + type_name: "SharedBoolSeries", + converter: |v| Value::Bool(v), + }) + } + StaticType::Record(layout) => { + let mut field_buffers = Vec::with_capacity(layout.fields.len()); + for (_, ty) in &layout.fields { + let member: Rc> = match ty { + StaticType::Float => { + Rc::new(RefCell::new(ScalarSeries::::new("FloatSeries"))) + } + StaticType::Int | StaticType::DateTime => { + Rc::new(RefCell::new(ScalarSeries::::new("IntSeries"))) + } + StaticType::Bool => { + Rc::new(RefCell::new(ScalarSeries::::new("BoolSeries"))) + } + _ => Rc::new(RefCell::new(ValueSeries::new())), + }; + field_buffers.push(member); + } + + let pusher = Rc::new(RefCell::new(RecordPusher { + field_buffers: field_buffers.clone(), + lookback: Some(100), + })); + pipe.borrow().add_observer(pusher); + + Rc::new(SharedRecordSeries { + layout: layout.clone(), + field_buffers, + }) + } + _ => { + let buffer = Rc::new(RefCell::new(RingBuffer::::new())); + let pusher = Rc::new(RefCell::new(ValuePusher { + buffer: buffer.clone(), + lookback: Some(100), + })); + pipe.borrow().add_observer(pusher); + Rc::new(SharedValueSeries { buffer }) + } + }; + + Rc::new(PipelineNode { + stream: pipe, + series, + }) +} + +// ============================================================================ +// Script Integration (RTL Registration) +// ============================================================================ + +use crate::ast::environment::Environment; +use crate::ast::types::{Keyword, Purity, RecordLayout, Signature, StaticType}; + +pub fn register(env: &Environment) { + // (create-random-ohlc seed limit) -> StreamNode + let generators = env.pipeline_generators.clone(); + + // Define the OHLC layout for typing + let ohlc_layout = RecordLayout::get_or_create(vec![ + (Keyword::intern("open"), StaticType::Float), + (Keyword::intern("high"), StaticType::Float), + (Keyword::intern("low"), StaticType::Float), + (Keyword::intern("close"), StaticType::Float), + ]); + + env.register_native_fn( + "create-random-ohlc", + StaticType::Function(Box::new(Signature { + params: StaticType::Tuple(vec![StaticType::Int, StaticType::Int]), + ret: StaticType::Series(Box::new(StaticType::Record(ohlc_layout.clone()))), + })), + Purity::Impure, // Modifies global generator registry + move |args: std::vec::Vec| { + if args.len() != 2 { + panic!("create-random-ohlc expects exactly 2 arguments (seed, limit)"); + } + let seed = if let Value::Int(s) = args[0] { + s as u64 + } else { + 0 + }; + let limit = if let Value::Int(l) = args[1] { + l as usize + } else { + 0 + }; + + // 1. Create the RootStream + let root_stream = Rc::new(RootStream::new()); + let stream_node = StreamNode { + inner: root_stream.clone(), + }; + + // 2. Setup the Layout for OHLC records + let layout = RecordLayout::get_or_create(vec![ + (Keyword::intern("open"), StaticType::Float), + (Keyword::intern("high"), StaticType::Float), + (Keyword::intern("low"), StaticType::Float), + (Keyword::intern("close"), StaticType::Float), + ]); + + // 3. Create the stateful generator closure + let mut current_tick = 0; + let mut last_close = 100.0; + + // We use a local PRNG instance for reproducibility based on the seed + let mut rng = fastrand::Rng::with_seed(seed); + + let generator = move || -> bool { + if current_tick >= limit { + return false; // Exhausted + } + + // Generate random OHLC (Random Walk) + let change = (rng.f64() - 0.5) * 2.0; + let open = last_close; + let high = open + (rng.f64() * 2.0).abs(); + let low = open - (rng.f64() * 2.0).abs(); + let close = open + change; + last_close = close; + + let record = Value::Record( + layout.clone(), + Rc::new(vec![ + Value::Float(open), + Value::Float(high), + Value::Float(low), + Value::Float(close), + ]), + ); + + // Pump the signal into the RootStream + root_stream.tick(record); + + current_tick += 1; + true // Still active + }; + + // 4. Register the generator in the Environment + generators.borrow_mut().push(Box::new(generator)); + + // 5. Return the stream reference to the script + Value::Object(Rc::new(stream_node)) + }, + ); + + // (create-ticker condition-closure) -> StreamNode + let ticker_generators = env.pipeline_generators.clone(); + let globals = env.global_values.clone(); + + env.register_native_fn( + "create-ticker", + StaticType::Function(Box::new(Signature { + params: StaticType::Tuple(vec![StaticType::Any]), // Expects a closure + ret: StaticType::Any, // Returns StreamNode + })), + Purity::Impure, + move |args: std::vec::Vec| { + if args.len() != 1 { + panic!("create-ticker expects exactly 1 argument (the condition closure)"); + } + + let closure_obj = if let Value::Object(obj) = &args[0] { + obj.clone() + } else { + panic!("create-ticker expects a closure as its argument"); + }; + + // 1. Create the RootStream + let root_stream = Rc::new(RootStream::new()); + let stream_node = StreamNode { + inner: root_stream.clone(), + }; + + // 2. Setup isolated VM + let mut ticker_vm = crate::ast::vm::VM::new(globals.clone()); + let my_closure = closure_obj.clone(); + + // 3. Create generator + let generator = move || -> bool { + match ticker_vm.run_with_args(my_closure.clone(), vec![]) { + Ok(Value::Bool(b)) => { + if b { + // Ticker pulses with a simple `true` value or `Void` + // We use true here so the pipe receives something tangible. + root_stream.tick(Value::Bool(true)); + true + } else { + false // Exhausted + } + } + Ok(_) => panic!("create-ticker closure must return a boolean"), + Err(e) => panic!("create-ticker closure execution failed: {}", e), + } + }; + + // 4. Register the generator + ticker_generators.borrow_mut().push(Box::new(generator)); + + // 5. Return stream + Value::Object(Rc::new(stream_node)) + }, + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::types::Value; + + #[test] + fn test_root_to_pipe_flow() { + let root = RootStream::new(); + let pipe = Rc::new(RefCell::new(PipeStream::new( + "test-pipe".to_string(), + 1, + None, + ))); + + root.add_observer(pipe.clone()); + + // Cycle 1: Root ticks 10.0 + root.tick(Value::Float(10.0)); + + let sig = pipe.borrow().current_signal().unwrap(); + assert_eq!(sig.cycle_id, 1); + if let Value::Float(v) = sig.value { + assert_eq!(v, 10.0); + } else { + panic!("Value must be Float(10.0)"); + } + + // Cycle 2: Root ticks 20.0 + root.tick(Value::Float(20.0)); + let sig2 = pipe.borrow().current_signal().unwrap(); + assert_eq!(sig2.cycle_id, 2); + if let Value::Float(v) = sig2.value { + assert_eq!(v, 20.0); + } else { + panic!("Value must be Float(20.0)"); + } + } + + #[test] + fn test_barrier_sync() { + // Pipe with 2 inputs + let pipe = Rc::new(RefCell::new(PipeStream::new( + "barrier-pipe".to_string(), + 2, + None, + ))); + + // Manual notifications simulate different input streams + pipe.borrow_mut().notify(0, 1, Value::Float(10.0)); + assert!( + pipe.borrow().current_signal().is_none(), + "Barrier should NOT be reached after 1st input" + ); + + pipe.borrow_mut().notify(1, 1, Value::Float(20.0)); + assert!( + pipe.borrow().current_signal().is_some(), + "Barrier SHOULD be reached after 2nd input" + ); + + let sig = pipe.borrow().current_signal().unwrap(); + assert_eq!(sig.cycle_id, 1); + } +} diff --git a/src/ast/rtl/type_registry.rs b/src/ast/rtl/type_registry.rs index 7882289..a9a75e7 100644 --- a/src/ast/rtl/type_registry.rs +++ b/src/ast/rtl/type_registry.rs @@ -202,7 +202,12 @@ mod tests { // Check static type let st = TypeRegistry::resolve_type::(®istry); if let StaticType::Record(layout) = st { - assert!(layout.fields.iter().any(|(k, _)| *k == Keyword::intern("greet"))); + assert!( + layout + .fields + .iter() + .any(|(k, _)| *k == Keyword::intern("greet")) + ); } else { panic!("Expected Record type"); } diff --git a/src/ast/types.rs b/src/ast/types.rs index d41f22c..2b3ffba 100644 --- a/src/ast/types.rs +++ b/src/ast/types.rs @@ -168,11 +168,7 @@ impl RecordLayout { return None; } let idx = self.fmap[offset]; - if idx < 0 { - None - } else { - Some(idx as usize) - } + if idx < 0 { None } else { Some(idx as usize) } } } @@ -180,7 +176,7 @@ impl RecordLayout { pub trait Object: fmt::Debug { fn type_name(&self) -> &'static str; fn as_any(&self) -> &dyn Any; - + /// Optional optimization: If this object behaves like a time series (indexable lookbacks), /// it can return a reference to its Series trait implementation. fn as_series(&self) -> Option<&dyn Series> { @@ -295,11 +291,11 @@ pub enum StaticType { DateTime, Text, Keyword, - Optional(Box), // Represents T | Void (e.g. for filter pipes) - List(Box), // Legacy / Dynamic list - Series(Box), // Time series of a specific type - Tuple(Vec), // Heterogeneous fixed-size - Vector(Box, usize), // Homogeneous fixed-size + Optional(Box), // Represents T | Void (e.g. for filter pipes) + List(Box), // Legacy / Dynamic list + Series(Box), // Time series of a specific type + Tuple(Vec), // Heterogeneous fixed-size + Vector(Box, usize), // Homogeneous fixed-size Matrix(Box, Vec), // Multi-dimensional homogeneous Record(std::sync::Arc), FieldAccessor(Keyword), @@ -371,7 +367,12 @@ impl fmt::Display for StaticType { impl StaticType { /// Returns true if `other` can be assigned to a location of type `self`. pub fn is_assignable_from(&self, other: &StaticType) -> bool { - if self == other || matches!(self, StaticType::Any) || matches!(other, StaticType::Any) || matches!(self, StaticType::Error) || matches!(other, StaticType::Error) { + if self == other + || matches!(self, StaticType::Any) + || matches!(other, StaticType::Any) + || matches!(self, StaticType::Error) + || matches!(other, StaticType::Error) + { return true; } @@ -411,9 +412,7 @@ impl StaticType { } } // Records are assignable if their layouts match (Structural identity via interning) - (StaticType::Record(a), StaticType::Record(b)) => { - std::sync::Arc::ptr_eq(a, b) - } + (StaticType::Record(a), StaticType::Record(b)) => std::sync::Arc::ptr_eq(a, b), // Series are assignable if their inner types are assignable (StaticType::Series(inner_a), StaticType::Series(inner_b)) => { inner_a.is_assignable_from(inner_b) @@ -550,7 +549,7 @@ impl Value { } Value::Record(layout, _) => StaticType::Record(layout.clone()), Value::FieldAccessor(k) => StaticType::FieldAccessor(*k), - Value::Function(_) => StaticType::Any, // Dynamic function + Value::Function(_) => StaticType::Any, // Dynamic function Value::Object(o) => StaticType::Object(o.type_name()), Value::Cell(c) => c.borrow().static_type(), Value::TailCallRequest(_) => StaticType::Any, // Internal state, but typable as Any @@ -596,10 +595,20 @@ impl fmt::Display for Value { Value::FieldAccessor(k) => write!(f, ".{}", k.name()), Value::Function(f_meta) => write!(f, "", f_meta.purity), Value::Object(o) => { - if let Some(pipe) = o.as_any().downcast_ref::() { + if let Some(pipe) = o + .as_any() + .downcast_ref::() + { write!(f, "PipelineNode[last: {:?}]", pipe.series.get_item(0)) - } else if let Some(val_series) = o.as_any().downcast_ref::() { - write!(f, "SharedValueSeries[len: {}]", val_series.buffer.borrow().len()) + } else if let Some(val_series) = + o.as_any() + .downcast_ref::() + { + write!( + f, + "SharedValueSeries[len: {}]", + val_series.buffer.borrow().len() + ) } else { write!(f, "<{}>", o.type_name()) } diff --git a/src/ast/vm.rs b/src/ast/vm.rs index fa30536..653d302 100644 --- a/src/ast/vm.rs +++ b/src/ast/vm.rs @@ -177,7 +177,11 @@ impl VM { } } - pub fn run_with_args(&mut self, closure_obj: Rc, args: Vec) -> Result { + pub fn run_with_args( + &mut self, + closure_obj: Rc, + args: Vec, + ) -> Result { let closure = closure_obj.as_any().downcast_ref::().unwrap(); self.stack.clear(); self.frames.clear(); @@ -323,29 +327,40 @@ impl VM { // 2. Downcast! We check at runtime if the pointer actually points to a `RecordSeries`. // `downcast_ref` is very fast (essentially an O(1) type ID comparison under the hood). - if let Some(record_series) = any_ptr.downcast_ref::() { - + if let Some(record_series) = + any_ptr.downcast_ref::() + { // 3. We call our highly performant 0-copy method on the series. - // It returns an `Rc>`, which is a shared pointer + // It returns an `Rc>`, which is a shared pointer // to the concrete column array (e.g., a `ScalarSeries`). if let Some(field_series) = record_series.field(*field) { // 4. We wrap this RefCell in our `SeriesView` struct. // The `SeriesView` acts as a pure `Object` for the VM, holding the reference. // CRITICAL: No array elements are copied here! This is pure, fast pointer juggling. // This single operation turns a SoA `RecordSeries` into a high-speed `FloatSeries` view. - let view = crate::ast::rtl::series::SeriesView::new(field_series, *field); + let view = + crate::ast::rtl::series::SeriesView::new(field_series, *field); return Ok(Value::Object(std::rc::Rc::new(view))); } else { - return Err(format!("RecordSeries does not have field :{}", field.name())); + return Err(format!( + "RecordSeries does not have field :{}", + field.name() + )); } } // Fallback if it's another type of object that is not a RecordSeries. - Err(format!("Attempt to access field on non-record object: {}", obj.type_name())) + Err(format!( + "Attempt to access field on non-record object: {}", + obj.type_name() + )) } // Error handling for primitives (Int, Float, etc.). - _ => Err(format!("Attempt to access field on non-record: {}", rec_val)), + _ => Err(format!( + "Attempt to access field on non-record: {}", + rec_val + )), } } @@ -413,11 +428,8 @@ impl VM { }); // Delegate to the RTL Factory for specialized buffer instantiation - let node = crate::ast::rtl::streams::build_pipeline_node( - obs_streams, - executor, - out_type, - ); + let node = + crate::ast::rtl::streams::build_pipeline_node(obs_streams, executor, out_type); Ok(Value::Object(node)) } @@ -489,23 +501,44 @@ impl VM { if let Some(idx) = layout.index_of(k) { return Ok(values[idx].clone()); } else { - return Err(format!("Record does not have field :{}", k.name())); + return Err(format!( + "Record does not have field :{}", + k.name() + )); } } else if let Value::Object(obj) = rec { // Polymorphic Field Access: Allow `.field` on a RecordSeries - if let Some(rs) = obj.as_any().downcast_ref::() { + if let Some(rs) = obj + .as_any() + .downcast_ref::() + { if let Some(field_series) = rs.field(k) { - let view = crate::ast::rtl::series::SeriesView::new(field_series, k); + let view = crate::ast::rtl::series::SeriesView::new( + field_series, + k, + ); return Ok(Value::Object(std::rc::Rc::new(view))); } else { - return Err(format!("RecordSeries does not have field :{}", k.name())); + return Err(format!( + "RecordSeries does not have field :{}", + k.name() + )); } } - return Err(format!("Field accessor .{} expects a record or RecordSeries, got {}", k.name(), obj.type_name())); + return Err(format!( + "Field accessor .{} expects a record or RecordSeries, got {}", + k.name(), + obj.type_name() + )); } else { - return Err(format!("Field accessor .{} expects a record or RecordSeries, got {}", k.name(), rec)); + return Err(format!( + "Field accessor .{} expects a record or RecordSeries, got {}", + k.name(), + rec + )); } - } _ => { + } + _ => { return Err(format!( "Tail call target is not a function: {}", func_val @@ -534,19 +567,37 @@ impl VM { } } else if let Value::Object(obj) = rec { // Polymorphic Field Access: Allow `.field` on a RecordSeries - if let Some(rs) = obj.as_any().downcast_ref::() { + if let Some(rs) = obj + .as_any() + .downcast_ref::() + { if let Some(field_series) = rs.field(k) { - let view = crate::ast::rtl::series::SeriesView::new(field_series, k); + let view = crate::ast::rtl::series::SeriesView::new( + field_series, + k, + ); break Ok(Value::Object(std::rc::Rc::new(view))); } else { - break Err(format!("RecordSeries does not have field :{}", k.name())); + break Err(format!( + "RecordSeries does not have field :{}", + k.name() + )); } } - break Err(format!("Field accessor .{} expects a record or RecordSeries, got {}", k.name(), obj.type_name())); + break Err(format!( + "Field accessor .{} expects a record or RecordSeries, got {}", + k.name(), + obj.type_name() + )); } else { - break Err(format!("Field accessor .{} expects a record or RecordSeries, got {}", k.name(), rec)); + break Err(format!( + "Field accessor .{} expects a record or RecordSeries, got {}", + k.name(), + rec + )); } - } Value::Object(obj) => { + } + Value::Object(obj) => { if let Some(closure) = obj.as_any().downcast_ref::() { let old_stack_top = self.stack.len(); self.frames.push(CallFrame { @@ -586,11 +637,17 @@ impl VM { // Unified Series Access (Step 1 of Dual Series Architecture) // This handles RecordSeries, SeriesView, and future SharedSeries polymorphically. if arg_vals.len() != 1 { - break Err(format!("{} indexer expects exactly 1 argument (the lookback index)", obj.type_name())); + break Err(format!( + "{} indexer expects exactly 1 argument (the lookback index)", + obj.type_name() + )); } if let Value::Int(idx) = arg_vals[0] { if idx < 0 { - break Err(format!("{} lookback index cannot be negative", obj.type_name())); + break Err(format!( + "{} lookback index cannot be negative", + obj.type_name() + )); } if let Some(val) = series.get_item(idx as usize) { break Ok(val); @@ -599,12 +656,16 @@ impl VM { break Ok(Value::Void); } } else { - break Err(format!("{} index must be an integer", obj.type_name())); + break Err(format!( + "{} index must be an integer", + obj.type_name() + )); } } else { break Err(format!("Object is not callable: {}", obj.type_name())); } - } _ => break Err(format!("Attempt to call non-function: {}", func_val)), + } + _ => break Err(format!("Attempt to call non-function: {}", func_val)), } } } @@ -651,7 +712,10 @@ impl VM { for v in values { evaluated_values.push(self.eval_internal(obs, v)?); } - Ok(Value::Record(layout.clone(), std::rc::Rc::new(evaluated_values))) + Ok(Value::Record( + layout.clone(), + std::rc::Rc::new(evaluated_values), + )) } BoundKind::Expansion { bound_expanded, .. } => self.eval_internal(obs, bound_expanded), BoundKind::Extension(ext) => Err(format!( diff --git a/src/bin/ast.rs b/src/bin/ast.rs index 8fd0df3..989f484 100644 --- a/src/bin/ast.rs +++ b/src/bin/ast.rs @@ -103,7 +103,12 @@ fn execute(env: &Environment, source: &str) { let result = env.compile(source); for diag in &result.diagnostics.items { let level = format!("{:?}", diag.level).to_uppercase(); - let loc = diag.identity.as_ref().and_then(|id| id.location).map(|l| format!(" at {}:{}", l.line, l.col)).unwrap_or_default(); + let loc = diag + .identity + .as_ref() + .and_then(|id| id.location) + .map(|l| format!(" at {}:{}", l.line, l.col)) + .unwrap_or_default(); eprintln!("{}{} : {}", level, loc, diag.message); } diff --git a/src/integration_test.rs b/src/integration_test.rs index e23eb39..f1e54c2 100644 --- a/src/integration_test.rs +++ b/src/integration_test.rs @@ -1,686 +1,759 @@ -#[cfg(test)] -mod tests { - use crate::ast::environment::Environment; - use crate::ast::nodes::UntypedKind; - use crate::ast::parser::Parser; - use crate::ast::types::Value; - - #[test] - fn test_parse_integer_constant() { - let source = "123"; - let mut parser = Parser::new(source); - let ast = parser.parse_expression(); - - if let UntypedKind::Constant(Value::Int(val)) = ast.kind { - assert_eq!(val, 123); - } else { - panic!("Expected Integer constant, got {:?}", ast.kind); - } - } - - #[test] - fn test_parse_negative_integer() { - let source = "-42"; - let mut parser = Parser::new(source); - let ast = parser.parse_expression(); - - if let UntypedKind::Constant(Value::Int(val)) = ast.kind { - assert_eq!(val, -42); - } else { - panic!("Expected Integer constant, got {:?}", ast.kind); - } - } - - #[test] - fn test_parse_float_constant() { - let source = "123.45"; - let mut parser = Parser::new(source); - let ast = parser.parse_expression(); - - if let UntypedKind::Constant(Value::Float(val)) = ast.kind { - assert_eq!(val, 123.45); - } else { - panic!("Expected Float constant, got {:?}", ast.kind); - } - } - - #[test] - fn test_parse_negative_float() { - let source = "-10.5"; - let mut parser = Parser::new(source); - let ast = parser.parse_expression(); - - if let UntypedKind::Constant(Value::Float(val)) = ast.kind { - assert_eq!(val, -10.5); - } else { - panic!("Expected Float constant, got {:?}", ast.kind); - } - } - - #[test] - fn test_closure_modification_from_source() { - let source = r#" - (do - (def x 10) - (def f (fn [] (assign x 20))) - (f) - x - ) - "#; - - let env = Environment::new(); - let compiled = env.compile(source).into_result().expect("Failed to compile"); - let linked = env.link(compiled); - let func = env.instantiate(linked); - let result: Result = Ok((func.func)(vec![])); - - match result { - Ok(Value::Int(20)) => (), - Ok(val) => panic!("Expected Int(20), got {:?}", val), - Err(e) => panic!("VM Error: {}", e), - } - } - - #[test] - fn test_examples() { - for opt in [false, true] { - let results = crate::utils::tester::run_functional_tests_with_optimization(opt); - for res in results { - assert!( - res.success, - "Example {} failed at opt {}: {}", - res.name, opt, res.message - ); - } - } - } - - #[test] - fn test_debug_mode_logging() { - let mut env = Environment::new(); - env.optimization = false; - let source = "(+ 10 20)"; - let result = env.run_debug(source).expect("Failed to run debug"); - - let (val, logs) = result; - - // 1. Check value - match val { - Ok(Value::Int(30)) => (), - _ => panic!("Expected Int(30), got {:?}", val), - } - - // 2. Check logs (should have entries for + and constants) - assert!(!logs.is_empty(), "Logs should not be empty"); - - // Look for typical trace patterns - let has_call = logs.iter().any(|l| l.contains("CALL")); - let has_const = logs.iter().any(|l| l.contains("CONST(10)")); - let has_result = logs.iter().any(|l| l.contains("} -> 30")); - - assert!(has_call, "Logs should contain CALL"); - assert!(has_const, "Logs should contain CONST(10)"); - assert!(has_result, "Logs should contain result 30"); - } - - #[test] - fn test_rtl_operators() { - let env = Environment::new(); - - // --- Arithmetic --- - assert_eq!(format!("{}", env.run_script("(+ 10 20)").unwrap()), "30"); - assert_eq!(format!("{}", env.run_script("(- 20 10)").unwrap()), "10"); - assert_eq!(format!("{}", env.run_script("(* 10 20)").unwrap()), "200"); - assert_eq!(format!("{}", env.run_script("(/ 20 10)").unwrap()), "2"); - assert_eq!(format!("{}", env.run_script("(// 20 3)").unwrap()), "6"); // 20 // 3 = 6 - assert_eq!(format!("{}", env.run_script("(% 20 3)").unwrap()), "2"); // 20 % 3 = 2 - - // --- Logic / Bitwise --- - assert_eq!( - format!("{}", env.run_script("(and true false)").unwrap()), - "false" - ); - assert_eq!( - format!("{}", env.run_script("(or true false)").unwrap()), - "true" - ); - assert_eq!( - format!("{}", env.run_script("(xor true false)").unwrap()), - "true" - ); - assert_eq!( - format!("{}", env.run_script("(not true)").unwrap()), - "false" - ); - - assert_eq!(format!("{}", env.run_script("(<< 1 2)").unwrap()), "4"); // 1 << 2 = 4 - assert_eq!(format!("{}", env.run_script("(>> 4 1)").unwrap()), "2"); // 4 >> 1 = 2 - assert_eq!(format!("{}", env.run_script("(and 3 1)").unwrap()), "1"); // 3 & 1 = 1 - - // --- Comparison --- - assert_eq!(format!("{}", env.run_script("(= 10 10)").unwrap()), "true"); - assert_eq!(format!("{}", env.run_script("(= 10 20)").unwrap()), "false"); - assert_eq!(format!("{}", env.run_script("(<> 10 20)").unwrap()), "true"); - assert_eq!(format!("{}", env.run_script("(< 10 20)").unwrap()), "true"); - assert_eq!(format!("{}", env.run_script("(> 10 20)").unwrap()), "false"); - assert_eq!(format!("{}", env.run_script("(<= 10 10)").unwrap()), "true"); - assert_eq!(format!("{}", env.run_script("(>= 10 10)").unwrap()), "true"); - - // --- NaN --- - assert_eq!(format!("{}", env.run_script("NaN").unwrap()), "NaN"); - } - - #[test] - fn test_random_isolation_between_environments() { - let env1 = Environment::new(); - let env2 = Environment::new(); - - // 1. Create a seeded generator in env1 - env1.run_script("(def rand (make-random 123))").unwrap(); - let val1_a = env1.run_script("(rand)").unwrap(); - - // 2. env2 should have its own default seed state for its generators - env2.run_script("(def rand (make-random))").unwrap(); - let val2_a = env2.run_script("(rand)").unwrap(); - - // They are highly unlikely to be equal by default, - // and seeding env1 MUST not have seeded env2. - assert_ne!( - val1_a, val2_a, - "Environments must have isolated PRNG states" - ); - - // 3. Create another generator in env2 with the same seed - env2.run_script("(def rand-same (make-random 123))").unwrap(); - let val2_b = env2.run_script("(rand-same)").unwrap(); - - // After same seeding, they should match (isolated but identical seed) - assert_eq!( - val1_a, val2_b, - "Different environments with the same seed must produce the same sequence" - ); - } - - #[test] - fn test_random_seeding_determinism() { - let env = Environment::new(); - - // 1. First run with seed 42 - env.run_script("(def rand1 (make-random 42))").unwrap(); - let val1 = env.run_script("(rand1)").unwrap(); - - // 2. Second run with same seed 42 - env.run_script("(def rand2 (make-random 42))").unwrap(); - let val2 = env.run_script("(rand2)").unwrap(); - - assert_eq!( - val1, val2, - "Random results must be identical for the same seed" - ); - - // 3. Third run with different seed - env.run_script("(def rand3 (make-random 123))").unwrap(); - let val3 = env.run_script("(rand3)").unwrap(); - assert_ne!(val1, val3, "Random results must differ for different seeds"); - } - - #[test] - fn test_now_function_not_folded() { - let env = Environment::new(); - let source = "(now)"; - - // 1. Check result type and value plausibility - let result = env.run_script(source).expect("Failed to run script"); - if let Value::DateTime(ts) = result { - let current = chrono::Utc::now().timestamp_millis(); - assert!(ts > 0); - assert!(ts <= current); - } else { - panic!("Expected DateTime, got {:?}", result); - } - - // 2. Verify it's NOT constant folded in the AST dump - let dump = env.dump_ast(source).expect("Failed to dump AST"); - assert!( - dump.contains("Call"), - "now() should remain a Call, not a Constant. Dump: \n{}", - dump - ); - assert!( - !dump.contains("Constant: #"), - "now() should NOT be folded into a specific timestamp constant. Dump: \n{}", - dump - ); - } - - #[test] - fn test_date_parsing() { - let env = Environment::new(); - let res = env.run_script("(date \"2023-01-01\")").unwrap(); - if let Value::DateTime(_) = res { - // OK - } else { - panic!("Expected DateTime, got {:?}", res); - } - } - - #[test] - #[should_panic(expected = "'again' is only allowed in tail position to avoid dead code.")] - fn test_again_non_tail_panic() { - let env = Environment::new(); - let source = "(do (def f (fn [x] (do (again (- x 1)) x))) (f 5))"; - // This will trigger the TCO pass which contains the validation logic - let _ = env.run_script(source); - } - - #[test] - fn test_dynamic_call_destructuring_underflow() { - let env = Environment::new(); - let source = "(do - (def call-dynamic (fn [f data] (f data))) - (def data [10 [20 30]]) - (def x (fn [[a [b c]]] (+ a (+ b c)))) - (call-dynamic x data))"; - - let result = env.run_script(source); - if let Err(e) = &result { - panic!("Failed: {}", e); - } - assert_eq!(format!("{}", result.unwrap()), "60"); - } - - #[test] - fn test_nested_destructuring_optimization() { - let env = Environment::new(); - - // 1. Tuple-to-Tuple - let source_tuple = "((fn [[x y]] (+ x y)) [10 20])"; - assert_eq!(format!("{}", env.run_script(source_tuple).unwrap()), "30"); - let dump_tuple = env.dump_ast(source_tuple).unwrap(); - assert!( - dump_tuple.contains("Constant: 30"), - "Nested tuple should be folded to 30. Dump:\n{}", - dump_tuple - ); - } - - #[test] - fn test_def_destructuring() { - let env = Environment::new(); - - // 1. Global destructuring - let source_global = "(do (def [a b] [1 2]) (+ a b))"; - assert_eq!(format!("{}", env.run_script(source_global).unwrap()), "3"); - - // 2. Local nested destructuring inside a function - let source_local = - "((fn [x] (do (def [a [[b c] d]] x) (+ a (+ b (+ c d))))) [1 [[2 3] 4]])"; - assert_eq!(format!("{}", env.run_script(source_local).unwrap()), "10"); - - // 3. Verify 'def' returns the assigned value - let source_return = "(def [x y] [7 8])"; - let res = env.run_script(source_return).unwrap(); - if let Value::Tuple(vals) = res { - assert_eq!(vals.len(), 2); - assert_eq!(format!("{}", vals[0]), "7"); - assert_eq!(format!("{}", vals[1]), "8"); - } else { - panic!("Expected tuple return from def, got {:?}", res); - } - } - - #[test] - fn test_assign_destructuring() { - // 1. Simple assignment destructuring - { - let env = Environment::new(); - let source_simple = "(do (def a 0) (def b 0) (assign [a b] [10 20]) (+ a b))"; - assert_eq!(format!("{}", env.run_script(source_simple).unwrap()), "30"); - } - - // 2. Nested assignment destructuring - { - let env = Environment::new(); - let source_nested = - "(do (def a 0) (def b 0) (def c 0) (assign [a [b c]] [1 [2 3]]) (+ a (+ b c)))"; - assert_eq!(format!("{}", env.run_script(source_nested).unwrap()), "6"); - } - - // 3. Assignment returns the assigned value - { - let env = Environment::new(); - let source_return = "(do (def a 0) (def b 0) (assign [a b] [5 6]))"; - let res = env.run_script(source_return).unwrap(); - if let Value::Tuple(vals) = res { - assert_eq!(vals.len(), 2); - assert_eq!(format!("{}", vals[0]), "5"); - assert_eq!(format!("{}", vals[1]), "6"); - } else { - panic!("Expected tuple return from assign, got {:?}", res); - } - } - } - - #[test] - fn test_pipeline_optional_type() { - let env = Environment::new(); - // The lambda uses an `if` without an `else` returning a float constant. - // The TypeChecker should deduce `Optional(Float)` for the lambda body, - // and correctly unwrap it to `Series(Float)` for the pipeline output. - let source = "(do - (def src (create-random-ohlc 42 10)) - (def filtered - (pipe [src] - (fn [tick] - (if true - 42.0 - ) - ) - ) - ) - filtered - )"; - let res = env.run_script(source); - if let Err(e) = &res { - panic!("Script failed to compile/run: {:?}", e); - } - - let val = res.unwrap(); - if let crate::ast::types::Value::Object(obj) = val { - assert_eq!(obj.type_name(), "PipelineNode"); - } else { - panic!("Expected an Object(PipelineNode)"); - } - } - - #[test] - fn test_multi_level_destructuring() { - let env = Environment::new(); - let source = "(do - (def process_data (fn [conf] - (do - (def [str s] conf) - (def [f ss] s) - [\"Symbol:\" str \"field:\" f \"id:\" ss] - ) - ) - ) - (process_data [\"btc\" [:close \"cls\"]]))"; - - let res = env.run_script(source).unwrap(); - assert_eq!( - format!("{}", res), - "[\"Symbol:\" \"btc\" \"field:\" :close \"id:\" \"cls\"]" - ); - } - - #[test] - fn test_closure_reassignment_optimization_bug() { - let env = Environment::new(); - // This test case reproduces a bug where the optimizer aggressively inlined a function - // ('f') even though its parameters ('x') were being assigned to in the body (by inner lambda). - // The fix ensures that such functions are NOT inlined. - let source = "(do (def f (fn [[x y]] (fn [] (assign x (+ x y))))) ((f [1 2])))"; - let res = env.run_script(source); - assert_eq!(format!("{}", res.unwrap()), "3"); - } - - #[test] - fn test_macro_inlining_identity_collision() { - let source = r#" - (do - (macro wrap [f] `(fn [x] (~f x))) - - (def add1 (fn [x] (+ x 1))) - (def add2 (fn [x] (+ x 2))) - - (def w1 (wrap add1)) - (def w2 (wrap add2)) - - (w1 (w2 10))) - "#; - - // 1. Verify the result is correct - let env_run = Environment::new(); - let res = env_run.run_script(source).expect("Failed to run script"); - assert_eq!(format!("{}", res), "13"); - - // 2. Verify that it was actually folded into a constant by the optimizer - let env_dump = Environment::new(); - let dump = env_dump.dump_ast(source).expect("Failed to dump AST"); - assert!( - dump.contains("Constant: 13"), - "Macro-wrapped calls should be fully folded to 13. Dump:\n{}", - dump - ); - // The definitions add1, add2, w1, w2 should be gone after dead code elimination - assert!( - !dump.contains("Define Variable"), - "Definitions should be removed by DCE" - ); - } - - #[test] - fn test_optimizer_upvalue_inlining_bug_repro() { - let env = Environment::new(); - let source = r#" - (do - (def make-counter (fn [init] - (do - (def val init) - { - :inc (fn [] (assign val (+ val 1))) - :get (fn [] val) - }))) - (def c (make-counter 10)) - ((.inc c)) - ((.get c))) - "#; - let res = env.run_script(source); - assert!(res.is_ok(), "Optimizer bug triggered: {:?}", res.err()); - assert_eq!(format!("{}", res.unwrap()), "11"); - } - - #[test] - fn test_optimizer_destructuring_inlining_and_mutation() { - let env = Environment::new(); - // 1. Test: Destructuring definition should allow inlining if not mutated - let source_inline = "(do (def [x y] [10 20]) (+ x y))"; - let res_inline = env.run_script(source_inline).unwrap(); - assert_eq!(format!("{}", res_inline), "30"); - - // 2. Test: Destructuring definition should NOT be inlined if mutated (Regression Test) - let source_mutation = r#" - (do - (def [a b] [1 2]) - (def f (fn [] (assign a (+ a b)))) - (f) - a) - "#; - let res_mutation = env.run_script(source_mutation).unwrap(); - assert_eq!(format!("{}", res_mutation), "3"); - } - - #[test] - fn test_record_basics() { - let env = Environment::new(); - let source = r#" - ((fn [user] [(.name user) (.age user)]) - {:name "Alice" :age 30}) - "#; - let res = env.run_script(source).unwrap(); - assert_eq!(format!("{}", res), "[\"Alice\" 30]"); - } - - #[test] - fn test_record_optimized_access() { - let env = Environment::new(); - let source_eval = "(.price {:id 1 :price 99.5})"; - - // 1. Check result (will be fully folded to a constant by the new optimization) - let res = env.run_script(source_eval).unwrap(); - assert_eq!(format!("{}", res), "99.5"); - - // 2. Verify optimization to GET_FIELD when the record contains non-constants - let source_ast = "(fn [id] (.price {:id id :price 99.5}))"; - let dump = env.dump_ast(source_ast).unwrap(); - assert!(dump.contains("GetField: .price"), "Should be optimized to GetField. Dump:\n{}", dump); - } - #[test] - fn test_first_class_field_accessor() { - let env = Environment::new(); - let source = r#" - (do - (def get-name .name) - ; Dynamic call to field accessor - (get-name {:name "Alice"})) - "#; - let res = env.run_script(source).unwrap(); - assert_eq!(format!("{}", res), "\"Alice\""); - } - - #[test] - fn test_record_constant_folding() { - let env = Environment::new(); - // Optimizer should fold (.x {:x 10}) into 10 - let source = "(.x {:x 10 :y 20})"; - let dump = env.dump_ast(source).unwrap(); - assert!(dump.contains("Constant: 10"), "Should evaluate GetField at compile time."); - } - - #[test] - fn test_record_literal_constant_folding() { - let env = Environment::new(); - let source = " {:a 1 :b 2} "; - let dump = env.dump_ast(source).unwrap(); - - // Ensure the record definition itself is folded into a Constant. - assert!(dump.contains("Constant: {:a 1, :b 2}"), "Should transform a pure record literal into a constant value."); - assert!(!dump.contains("Record {"), "Should not leave a runtime Record node in the AST."); - } - - #[test] - fn test_record_inlining_in_while_loop() { - let env_ast = Environment::new(); - // Ensure that constants (like the config record) are inherited into inner lambda scopes (like the body of a while loop). - let source = r#" - (do - (def loop-config {:start 0 :limit 10}) - (def loop-idx 0) - (while (< loop-idx (.limit loop-config)) - (assign loop-idx (+ loop-idx 1))) - loop-idx - ) - "#; - let dump = env_ast.dump_ast(source).unwrap(); - - // The optimizer should inline `loop-config`, resolving `(.limit loop-config)` to `10`. - // The GetField and the Get for 'loop-config' should vanish inside the condition. - assert!(!dump.contains("GetField: .limit"), "The record field should be completely inlined."); - assert!(dump.contains("Constant: 10"), "The limit should be resolved to a constant 10."); - - // The result of running it should obviously still be correct. - let env_run = Environment::new(); - let res = env_run.run_script(source).unwrap(); - assert_eq!(format!("{}", res), "10"); - } - - #[test] - fn test_record_errors() { - let env = Environment::new(); - - // Both cases are reported by the TypeChecker since it can't resolve the call - // for a missing field or a non-record argument. - - // 1. Missing field - let res_missing = env.run_script("(.missing {:a 1})"); - assert!(res_missing.is_err()); - assert!(res_missing.unwrap_err().contains("Invalid arguments")); - - // 2. Not a record - let res_not_rec = env.run_script("(.name 123)"); - assert!(res_not_rec.is_err()); - assert!(res_not_rec.unwrap_err().contains("Invalid arguments")); - } - - #[test] - fn test_record_layout_interning() { - let env = Environment::new(); - let source = r#" - (do - (def r1 {:a 1 :b 2}) - (def r2 {:a 10 :b 20}) - ; Identical layouts result in identical types - (= r1 r2)) - "#; - // This will be false because values differ, but let's just check if it compiles and runs. - // To really test interning, we'd need a way to check if layouts are the same Arc. - let res = env.run_script(source).unwrap(); - assert_eq!(format!("{}", res), "false"); - } - - #[test] - fn test_error_recovery_parser() { - let env = Environment::new(); - // Syntax error: mismatched bracket/paren - let source = "(do (def a [1 2 ) )"; - let result = env.compile(source); - - assert!(result.diagnostics.has_errors(), "Expected parser errors"); - let error_msgs: Vec<_> = result.diagnostics.items.iter().map(|d| d.message.as_str()).collect(); - - // It should complain about finding ) instead of ] - assert!(error_msgs.iter().any(|m| m.contains("RightParen")), "Expected error about RightParen"); - - // AST should still be partially built (not None) - assert!(result.ast.is_some(), "AST should be partially built despite parser errors"); - } - - #[test] - fn test_error_recovery_binder() { - let env = Environment::new(); - // Semantic error: undefined variable - let source = "(do (def a 10) (def b unknown_var) (+ a 5))"; - let result = env.compile(source); - - assert!(result.diagnostics.has_errors(), "Expected binder errors"); - let error_msgs: Vec<_> = result.diagnostics.items.iter().map(|d| d.message.as_str()).collect(); - - assert!(error_msgs.iter().any(|m| m.contains("Undefined variable 'unknown_var'")), "Expected undefined variable error"); - assert!(result.ast.is_some(), "AST should be built with Error nodes"); - } - - #[test] - fn test_error_recovery_type_checker() { - let env = Environment::new(); - // Semantic error: Type mismatch in function call - let source = "(do (def a 10) (def b (not \"text\")) (- a 2))"; - let result = env.compile(source); - - assert!(result.diagnostics.has_errors(), "Expected type checker errors"); - let error_msgs: Vec<_> = result.diagnostics.items.iter().map(|d| d.message.as_str()).collect(); - - assert!(error_msgs.iter().any(|m| m.contains("Invalid arguments for function call")), "Expected invalid arguments error"); - assert!(result.ast.is_some(), "AST should be built with Error nodes"); - } - - #[test] - fn test_error_recovery_multiple_errors() { - let env = Environment::new(); - // A script with multiple errors that should all be collected - let source = r#" - (do - (def a undefined_var) - (def b (not "text")) - ) - "#; - let result = env.compile(source); - - assert!(result.diagnostics.has_errors()); - assert!(result.diagnostics.items.len() >= 2, "Expected multiple errors to be collected"); - - let error_msgs: Vec<_> = result.diagnostics.items.iter().map(|d| d.message.as_str()).collect(); - assert!(error_msgs.iter().any(|m| m.contains("Undefined variable 'undefined_var'"))); - assert!(error_msgs.iter().any(|m| m.contains("Invalid arguments for function call"))); - } -} +#[cfg(test)] +mod tests { + use crate::ast::environment::Environment; + use crate::ast::nodes::UntypedKind; + use crate::ast::parser::Parser; + use crate::ast::types::Value; + + #[test] + fn test_parse_integer_constant() { + let source = "123"; + let mut parser = Parser::new(source); + let ast = parser.parse_expression(); + + if let UntypedKind::Constant(Value::Int(val)) = ast.kind { + assert_eq!(val, 123); + } else { + panic!("Expected Integer constant, got {:?}", ast.kind); + } + } + + #[test] + fn test_parse_negative_integer() { + let source = "-42"; + let mut parser = Parser::new(source); + let ast = parser.parse_expression(); + + if let UntypedKind::Constant(Value::Int(val)) = ast.kind { + assert_eq!(val, -42); + } else { + panic!("Expected Integer constant, got {:?}", ast.kind); + } + } + + #[test] + fn test_parse_float_constant() { + let source = "123.45"; + let mut parser = Parser::new(source); + let ast = parser.parse_expression(); + + if let UntypedKind::Constant(Value::Float(val)) = ast.kind { + assert_eq!(val, 123.45); + } else { + panic!("Expected Float constant, got {:?}", ast.kind); + } + } + + #[test] + fn test_parse_negative_float() { + let source = "-10.5"; + let mut parser = Parser::new(source); + let ast = parser.parse_expression(); + + if let UntypedKind::Constant(Value::Float(val)) = ast.kind { + assert_eq!(val, -10.5); + } else { + panic!("Expected Float constant, got {:?}", ast.kind); + } + } + + #[test] + fn test_closure_modification_from_source() { + let source = r#" + (do + (def x 10) + (def f (fn [] (assign x 20))) + (f) + x + ) + "#; + + let env = Environment::new(); + let compiled = env + .compile(source) + .into_result() + .expect("Failed to compile"); + let linked = env.link(compiled); + let func = env.instantiate(linked); + let result: Result = Ok((func.func)(vec![])); + + match result { + Ok(Value::Int(20)) => (), + Ok(val) => panic!("Expected Int(20), got {:?}", val), + Err(e) => panic!("VM Error: {}", e), + } + } + + #[test] + fn test_examples() { + for opt in [false, true] { + let results = crate::utils::tester::run_functional_tests_with_optimization(opt); + for res in results { + assert!( + res.success, + "Example {} failed at opt {}: {}", + res.name, opt, res.message + ); + } + } + } + + #[test] + fn test_debug_mode_logging() { + let mut env = Environment::new(); + env.optimization = false; + let source = "(+ 10 20)"; + let result = env.run_debug(source).expect("Failed to run debug"); + + let (val, logs) = result; + + // 1. Check value + match val { + Ok(Value::Int(30)) => (), + _ => panic!("Expected Int(30), got {:?}", val), + } + + // 2. Check logs (should have entries for + and constants) + assert!(!logs.is_empty(), "Logs should not be empty"); + + // Look for typical trace patterns + let has_call = logs.iter().any(|l| l.contains("CALL")); + let has_const = logs.iter().any(|l| l.contains("CONST(10)")); + let has_result = logs.iter().any(|l| l.contains("} -> 30")); + + assert!(has_call, "Logs should contain CALL"); + assert!(has_const, "Logs should contain CONST(10)"); + assert!(has_result, "Logs should contain result 30"); + } + + #[test] + fn test_rtl_operators() { + let env = Environment::new(); + + // --- Arithmetic --- + assert_eq!(format!("{}", env.run_script("(+ 10 20)").unwrap()), "30"); + assert_eq!(format!("{}", env.run_script("(- 20 10)").unwrap()), "10"); + assert_eq!(format!("{}", env.run_script("(* 10 20)").unwrap()), "200"); + assert_eq!(format!("{}", env.run_script("(/ 20 10)").unwrap()), "2"); + assert_eq!(format!("{}", env.run_script("(// 20 3)").unwrap()), "6"); // 20 // 3 = 6 + assert_eq!(format!("{}", env.run_script("(% 20 3)").unwrap()), "2"); // 20 % 3 = 2 + + // --- Logic / Bitwise --- + assert_eq!( + format!("{}", env.run_script("(and true false)").unwrap()), + "false" + ); + assert_eq!( + format!("{}", env.run_script("(or true false)").unwrap()), + "true" + ); + assert_eq!( + format!("{}", env.run_script("(xor true false)").unwrap()), + "true" + ); + assert_eq!( + format!("{}", env.run_script("(not true)").unwrap()), + "false" + ); + + assert_eq!(format!("{}", env.run_script("(<< 1 2)").unwrap()), "4"); // 1 << 2 = 4 + assert_eq!(format!("{}", env.run_script("(>> 4 1)").unwrap()), "2"); // 4 >> 1 = 2 + assert_eq!(format!("{}", env.run_script("(and 3 1)").unwrap()), "1"); // 3 & 1 = 1 + + // --- Comparison --- + assert_eq!(format!("{}", env.run_script("(= 10 10)").unwrap()), "true"); + assert_eq!(format!("{}", env.run_script("(= 10 20)").unwrap()), "false"); + assert_eq!(format!("{}", env.run_script("(<> 10 20)").unwrap()), "true"); + assert_eq!(format!("{}", env.run_script("(< 10 20)").unwrap()), "true"); + assert_eq!(format!("{}", env.run_script("(> 10 20)").unwrap()), "false"); + assert_eq!(format!("{}", env.run_script("(<= 10 10)").unwrap()), "true"); + assert_eq!(format!("{}", env.run_script("(>= 10 10)").unwrap()), "true"); + + // --- NaN --- + assert_eq!(format!("{}", env.run_script("NaN").unwrap()), "NaN"); + } + + #[test] + fn test_random_isolation_between_environments() { + let env1 = Environment::new(); + let env2 = Environment::new(); + + // 1. Create a seeded generator in env1 + env1.run_script("(def rand (make-random 123))").unwrap(); + let val1_a = env1.run_script("(rand)").unwrap(); + + // 2. env2 should have its own default seed state for its generators + env2.run_script("(def rand (make-random))").unwrap(); + let val2_a = env2.run_script("(rand)").unwrap(); + + // They are highly unlikely to be equal by default, + // and seeding env1 MUST not have seeded env2. + assert_ne!( + val1_a, val2_a, + "Environments must have isolated PRNG states" + ); + + // 3. Create another generator in env2 with the same seed + env2.run_script("(def rand-same (make-random 123))") + .unwrap(); + let val2_b = env2.run_script("(rand-same)").unwrap(); + + // After same seeding, they should match (isolated but identical seed) + assert_eq!( + val1_a, val2_b, + "Different environments with the same seed must produce the same sequence" + ); + } + + #[test] + fn test_random_seeding_determinism() { + let env = Environment::new(); + + // 1. First run with seed 42 + env.run_script("(def rand1 (make-random 42))").unwrap(); + let val1 = env.run_script("(rand1)").unwrap(); + + // 2. Second run with same seed 42 + env.run_script("(def rand2 (make-random 42))").unwrap(); + let val2 = env.run_script("(rand2)").unwrap(); + + assert_eq!( + val1, val2, + "Random results must be identical for the same seed" + ); + + // 3. Third run with different seed + env.run_script("(def rand3 (make-random 123))").unwrap(); + let val3 = env.run_script("(rand3)").unwrap(); + assert_ne!(val1, val3, "Random results must differ for different seeds"); + } + + #[test] + fn test_now_function_not_folded() { + let env = Environment::new(); + let source = "(now)"; + + // 1. Check result type and value plausibility + let result = env.run_script(source).expect("Failed to run script"); + if let Value::DateTime(ts) = result { + let current = chrono::Utc::now().timestamp_millis(); + assert!(ts > 0); + assert!(ts <= current); + } else { + panic!("Expected DateTime, got {:?}", result); + } + + // 2. Verify it's NOT constant folded in the AST dump + let dump = env.dump_ast(source).expect("Failed to dump AST"); + assert!( + dump.contains("Call"), + "now() should remain a Call, not a Constant. Dump: \n{}", + dump + ); + assert!( + !dump.contains("Constant: #"), + "now() should NOT be folded into a specific timestamp constant. Dump: \n{}", + dump + ); + } + + #[test] + fn test_date_parsing() { + let env = Environment::new(); + let res = env.run_script("(date \"2023-01-01\")").unwrap(); + if let Value::DateTime(_) = res { + // OK + } else { + panic!("Expected DateTime, got {:?}", res); + } + } + + #[test] + #[should_panic(expected = "'again' is only allowed in tail position to avoid dead code.")] + fn test_again_non_tail_panic() { + let env = Environment::new(); + let source = "(do (def f (fn [x] (do (again (- x 1)) x))) (f 5))"; + // This will trigger the TCO pass which contains the validation logic + let _ = env.run_script(source); + } + + #[test] + fn test_dynamic_call_destructuring_underflow() { + let env = Environment::new(); + let source = "(do + (def call-dynamic (fn [f data] (f data))) + (def data [10 [20 30]]) + (def x (fn [[a [b c]]] (+ a (+ b c)))) + (call-dynamic x data))"; + + let result = env.run_script(source); + if let Err(e) = &result { + panic!("Failed: {}", e); + } + assert_eq!(format!("{}", result.unwrap()), "60"); + } + + #[test] + fn test_nested_destructuring_optimization() { + let env = Environment::new(); + + // 1. Tuple-to-Tuple + let source_tuple = "((fn [[x y]] (+ x y)) [10 20])"; + assert_eq!(format!("{}", env.run_script(source_tuple).unwrap()), "30"); + let dump_tuple = env.dump_ast(source_tuple).unwrap(); + assert!( + dump_tuple.contains("Constant: 30"), + "Nested tuple should be folded to 30. Dump:\n{}", + dump_tuple + ); + } + + #[test] + fn test_def_destructuring() { + let env = Environment::new(); + + // 1. Global destructuring + let source_global = "(do (def [a b] [1 2]) (+ a b))"; + assert_eq!(format!("{}", env.run_script(source_global).unwrap()), "3"); + + // 2. Local nested destructuring inside a function + let source_local = + "((fn [x] (do (def [a [[b c] d]] x) (+ a (+ b (+ c d))))) [1 [[2 3] 4]])"; + assert_eq!(format!("{}", env.run_script(source_local).unwrap()), "10"); + + // 3. Verify 'def' returns the assigned value + let source_return = "(def [x y] [7 8])"; + let res = env.run_script(source_return).unwrap(); + if let Value::Tuple(vals) = res { + assert_eq!(vals.len(), 2); + assert_eq!(format!("{}", vals[0]), "7"); + assert_eq!(format!("{}", vals[1]), "8"); + } else { + panic!("Expected tuple return from def, got {:?}", res); + } + } + + #[test] + fn test_assign_destructuring() { + // 1. Simple assignment destructuring + { + let env = Environment::new(); + let source_simple = "(do (def a 0) (def b 0) (assign [a b] [10 20]) (+ a b))"; + assert_eq!(format!("{}", env.run_script(source_simple).unwrap()), "30"); + } + + // 2. Nested assignment destructuring + { + let env = Environment::new(); + let source_nested = + "(do (def a 0) (def b 0) (def c 0) (assign [a [b c]] [1 [2 3]]) (+ a (+ b c)))"; + assert_eq!(format!("{}", env.run_script(source_nested).unwrap()), "6"); + } + + // 3. Assignment returns the assigned value + { + let env = Environment::new(); + let source_return = "(do (def a 0) (def b 0) (assign [a b] [5 6]))"; + let res = env.run_script(source_return).unwrap(); + if let Value::Tuple(vals) = res { + assert_eq!(vals.len(), 2); + assert_eq!(format!("{}", vals[0]), "5"); + assert_eq!(format!("{}", vals[1]), "6"); + } else { + panic!("Expected tuple return from assign, got {:?}", res); + } + } + } + + #[test] + fn test_pipeline_optional_type() { + let env = Environment::new(); + // The lambda uses an `if` without an `else` returning a float constant. + // The TypeChecker should deduce `Optional(Float)` for the lambda body, + // and correctly unwrap it to `Series(Float)` for the pipeline output. + let source = "(do + (def src (create-random-ohlc 42 10)) + (def filtered + (pipe [src] + (fn [tick] + (if true + 42.0 + ) + ) + ) + ) + filtered + )"; + let res = env.run_script(source); + if let Err(e) = &res { + panic!("Script failed to compile/run: {:?}", e); + } + + let val = res.unwrap(); + if let crate::ast::types::Value::Object(obj) = val { + assert_eq!(obj.type_name(), "PipelineNode"); + } else { + panic!("Expected an Object(PipelineNode)"); + } + } + + #[test] + fn test_multi_level_destructuring() { + let env = Environment::new(); + let source = "(do + (def process_data (fn [conf] + (do + (def [str s] conf) + (def [f ss] s) + [\"Symbol:\" str \"field:\" f \"id:\" ss] + ) + ) + ) + (process_data [\"btc\" [:close \"cls\"]]))"; + + let res = env.run_script(source).unwrap(); + assert_eq!( + format!("{}", res), + "[\"Symbol:\" \"btc\" \"field:\" :close \"id:\" \"cls\"]" + ); + } + + #[test] + fn test_closure_reassignment_optimization_bug() { + let env = Environment::new(); + // This test case reproduces a bug where the optimizer aggressively inlined a function + // ('f') even though its parameters ('x') were being assigned to in the body (by inner lambda). + // The fix ensures that such functions are NOT inlined. + let source = "(do (def f (fn [[x y]] (fn [] (assign x (+ x y))))) ((f [1 2])))"; + let res = env.run_script(source); + assert_eq!(format!("{}", res.unwrap()), "3"); + } + + #[test] + fn test_macro_inlining_identity_collision() { + let source = r#" + (do + (macro wrap [f] `(fn [x] (~f x))) + + (def add1 (fn [x] (+ x 1))) + (def add2 (fn [x] (+ x 2))) + + (def w1 (wrap add1)) + (def w2 (wrap add2)) + + (w1 (w2 10))) + "#; + + // 1. Verify the result is correct + let env_run = Environment::new(); + let res = env_run.run_script(source).expect("Failed to run script"); + assert_eq!(format!("{}", res), "13"); + + // 2. Verify that it was actually folded into a constant by the optimizer + let env_dump = Environment::new(); + let dump = env_dump.dump_ast(source).expect("Failed to dump AST"); + assert!( + dump.contains("Constant: 13"), + "Macro-wrapped calls should be fully folded to 13. Dump:\n{}", + dump + ); + // The definitions add1, add2, w1, w2 should be gone after dead code elimination + assert!( + !dump.contains("Define Variable"), + "Definitions should be removed by DCE" + ); + } + + #[test] + fn test_optimizer_upvalue_inlining_bug_repro() { + let env = Environment::new(); + let source = r#" + (do + (def make-counter (fn [init] + (do + (def val init) + { + :inc (fn [] (assign val (+ val 1))) + :get (fn [] val) + }))) + (def c (make-counter 10)) + ((.inc c)) + ((.get c))) + "#; + let res = env.run_script(source); + assert!(res.is_ok(), "Optimizer bug triggered: {:?}", res.err()); + assert_eq!(format!("{}", res.unwrap()), "11"); + } + + #[test] + fn test_optimizer_destructuring_inlining_and_mutation() { + let env = Environment::new(); + // 1. Test: Destructuring definition should allow inlining if not mutated + let source_inline = "(do (def [x y] [10 20]) (+ x y))"; + let res_inline = env.run_script(source_inline).unwrap(); + assert_eq!(format!("{}", res_inline), "30"); + + // 2. Test: Destructuring definition should NOT be inlined if mutated (Regression Test) + let source_mutation = r#" + (do + (def [a b] [1 2]) + (def f (fn [] (assign a (+ a b)))) + (f) + a) + "#; + let res_mutation = env.run_script(source_mutation).unwrap(); + assert_eq!(format!("{}", res_mutation), "3"); + } + + #[test] + fn test_record_basics() { + let env = Environment::new(); + let source = r#" + ((fn [user] [(.name user) (.age user)]) + {:name "Alice" :age 30}) + "#; + let res = env.run_script(source).unwrap(); + assert_eq!(format!("{}", res), "[\"Alice\" 30]"); + } + + #[test] + fn test_record_optimized_access() { + let env = Environment::new(); + let source_eval = "(.price {:id 1 :price 99.5})"; + + // 1. Check result (will be fully folded to a constant by the new optimization) + let res = env.run_script(source_eval).unwrap(); + assert_eq!(format!("{}", res), "99.5"); + + // 2. Verify optimization to GET_FIELD when the record contains non-constants + let source_ast = "(fn [id] (.price {:id id :price 99.5}))"; + let dump = env.dump_ast(source_ast).unwrap(); + assert!( + dump.contains("GetField: .price"), + "Should be optimized to GetField. Dump:\n{}", + dump + ); + } + #[test] + fn test_first_class_field_accessor() { + let env = Environment::new(); + let source = r#" + (do + (def get-name .name) + ; Dynamic call to field accessor + (get-name {:name "Alice"})) + "#; + let res = env.run_script(source).unwrap(); + assert_eq!(format!("{}", res), "\"Alice\""); + } + + #[test] + fn test_record_constant_folding() { + let env = Environment::new(); + // Optimizer should fold (.x {:x 10}) into 10 + let source = "(.x {:x 10 :y 20})"; + let dump = env.dump_ast(source).unwrap(); + assert!( + dump.contains("Constant: 10"), + "Should evaluate GetField at compile time." + ); + } + + #[test] + fn test_record_literal_constant_folding() { + let env = Environment::new(); + let source = " {:a 1 :b 2} "; + let dump = env.dump_ast(source).unwrap(); + + // Ensure the record definition itself is folded into a Constant. + assert!( + dump.contains("Constant: {:a 1, :b 2}"), + "Should transform a pure record literal into a constant value." + ); + assert!( + !dump.contains("Record {"), + "Should not leave a runtime Record node in the AST." + ); + } + + #[test] + fn test_record_inlining_in_while_loop() { + let env_ast = Environment::new(); + // Ensure that constants (like the config record) are inherited into inner lambda scopes (like the body of a while loop). + let source = r#" + (do + (def loop-config {:start 0 :limit 10}) + (def loop-idx 0) + (while (< loop-idx (.limit loop-config)) + (assign loop-idx (+ loop-idx 1))) + loop-idx + ) + "#; + let dump = env_ast.dump_ast(source).unwrap(); + + // The optimizer should inline `loop-config`, resolving `(.limit loop-config)` to `10`. + // The GetField and the Get for 'loop-config' should vanish inside the condition. + assert!( + !dump.contains("GetField: .limit"), + "The record field should be completely inlined." + ); + assert!( + dump.contains("Constant: 10"), + "The limit should be resolved to a constant 10." + ); + + // The result of running it should obviously still be correct. + let env_run = Environment::new(); + let res = env_run.run_script(source).unwrap(); + assert_eq!(format!("{}", res), "10"); + } + + #[test] + fn test_record_errors() { + let env = Environment::new(); + + // Both cases are reported by the TypeChecker since it can't resolve the call + // for a missing field or a non-record argument. + + // 1. Missing field + let res_missing = env.run_script("(.missing {:a 1})"); + assert!(res_missing.is_err()); + assert!(res_missing.unwrap_err().contains("Invalid arguments")); + + // 2. Not a record + let res_not_rec = env.run_script("(.name 123)"); + assert!(res_not_rec.is_err()); + assert!(res_not_rec.unwrap_err().contains("Invalid arguments")); + } + + #[test] + fn test_record_layout_interning() { + let env = Environment::new(); + let source = r#" + (do + (def r1 {:a 1 :b 2}) + (def r2 {:a 10 :b 20}) + ; Identical layouts result in identical types + (= r1 r2)) + "#; + // This will be false because values differ, but let's just check if it compiles and runs. + // To really test interning, we'd need a way to check if layouts are the same Arc. + let res = env.run_script(source).unwrap(); + assert_eq!(format!("{}", res), "false"); + } + + #[test] + fn test_error_recovery_parser() { + let env = Environment::new(); + // Syntax error: mismatched bracket/paren + let source = "(do (def a [1 2 ) )"; + let result = env.compile(source); + + assert!(result.diagnostics.has_errors(), "Expected parser errors"); + let error_msgs: Vec<_> = result + .diagnostics + .items + .iter() + .map(|d| d.message.as_str()) + .collect(); + + // It should complain about finding ) instead of ] + assert!( + error_msgs.iter().any(|m| m.contains("RightParen")), + "Expected error about RightParen" + ); + + // AST should still be partially built (not None) + assert!( + result.ast.is_some(), + "AST should be partially built despite parser errors" + ); + } + + #[test] + fn test_error_recovery_binder() { + let env = Environment::new(); + // Semantic error: undefined variable + let source = "(do (def a 10) (def b unknown_var) (+ a 5))"; + let result = env.compile(source); + + assert!(result.diagnostics.has_errors(), "Expected binder errors"); + let error_msgs: Vec<_> = result + .diagnostics + .items + .iter() + .map(|d| d.message.as_str()) + .collect(); + + assert!( + error_msgs + .iter() + .any(|m| m.contains("Undefined variable 'unknown_var'")), + "Expected undefined variable error" + ); + assert!(result.ast.is_some(), "AST should be built with Error nodes"); + } + + #[test] + fn test_error_recovery_type_checker() { + let env = Environment::new(); + // Semantic error: Type mismatch in function call + let source = "(do (def a 10) (def b (not \"text\")) (- a 2))"; + let result = env.compile(source); + + assert!( + result.diagnostics.has_errors(), + "Expected type checker errors" + ); + let error_msgs: Vec<_> = result + .diagnostics + .items + .iter() + .map(|d| d.message.as_str()) + .collect(); + + assert!( + error_msgs + .iter() + .any(|m| m.contains("Invalid arguments for function call")), + "Expected invalid arguments error" + ); + assert!(result.ast.is_some(), "AST should be built with Error nodes"); + } + + #[test] + fn test_error_recovery_multiple_errors() { + let env = Environment::new(); + // A script with multiple errors that should all be collected + let source = r#" + (do + (def a undefined_var) + (def b (not "text")) + ) + "#; + let result = env.compile(source); + + assert!(result.diagnostics.has_errors()); + assert!( + result.diagnostics.items.len() >= 2, + "Expected multiple errors to be collected" + ); + + let error_msgs: Vec<_> = result + .diagnostics + .items + .iter() + .map(|d| d.message.as_str()) + .collect(); + assert!( + error_msgs + .iter() + .any(|m| m.contains("Undefined variable 'undefined_var'")) + ); + assert!( + error_msgs + .iter() + .any(|m| m.contains("Invalid arguments for function call")) + ); + } +}