use std::any::Any; use std::collections::VecDeque; use std::fmt::{self, Debug}; use std::rc::Rc; use std::sync::Arc; use crate::ast::environment::Environment; use crate::ast::types::{Keyword, Object, RecordLayout, Series, StaticType, Value}; use crate::ast::types::{Purity, Signature}; // ============================================================================ // 1. Core Storage Engine // ============================================================================ /// A generic, efficient ring buffer for time series data. /// (Replaces Delphi's TChunkArray for now with VecDeque for O(1) push/pop) #[derive(Clone)] pub struct RingBuffer { buffer: VecDeque, total_count: u64, lookback_limit: usize, } impl RingBuffer { pub fn new(lookback_limit: usize) -> Self { // Pre-allocate up to 500 elements to minimize heap fragmentation let initial_capacity = std::cmp::min(lookback_limit, 500); Self { buffer: VecDeque::with_capacity(initial_capacity), total_count: 0, lookback_limit, } } /// Adds an element. Removes the oldest if the lookback limit is reached. #[inline] pub fn push(&mut self, item: T) { self.buffer.push_back(item); self.total_count += 1; while self.buffer.len() > self.lookback_limit { self.buffer.pop_front(); } } /// Access in "Financial Style": Index 0 is the newest element. #[inline] pub fn get(&self, index: usize) -> Option<&T> { let len = self.buffer.len(); if index < len { self.buffer.get(len - 1 - index) } else { None } } #[inline] pub fn total_count(&self) -> u64 { self.total_count } #[inline] pub fn len(&self) -> usize { self.buffer.len() } pub fn is_empty(&self) -> bool { self.buffer.is_empty() } } // ============================================================================ // 2. Trait & Primitives for Fast-Paths // ============================================================================ /// Marker trait ensuring our fast arrays only store true, flat scalars /// (No pointers/heaps like String or Record). pub trait ScalarValue: Copy + Debug + 'static { fn to_value(self) -> Value; /// Converts a dynamic `Value` back to this scalar type, returning `None` on mismatch. fn from_value(v: Value) -> Option; } impl ScalarValue for f64 { fn to_value(self) -> Value { Value::Float(self) } fn from_value(v: Value) -> Option { v.as_float() } } impl ScalarValue for i64 { fn to_value(self) -> Value { Value::Int(self) } fn from_value(v: Value) -> Option { v.as_int() } } impl ScalarValue for bool { fn to_value(self) -> Value { Value::Bool(self) } fn from_value(v: Value) -> Option { if let Value::Bool(b) = v { Some(b) } else { None } } } /// A highly optimized, type-safe series for primitive values (Float, Int, Bool). #[derive(Clone)] pub struct ScalarSeries { pub data: std::cell::RefCell>, pub type_name: &'static str, } impl ScalarSeries { pub fn new(type_name: &'static str, lookback_limit: usize) -> Self { Self { data: std::cell::RefCell::new(RingBuffer::new(lookback_limit)), type_name, } } } impl Debug for ScalarSeries { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, "ScalarSeries<{}>[len: {}]", self.type_name, self.data.borrow().len() ) } } // Makes the series usable as a dynamic Object for the VM. impl Object for ScalarSeries { fn type_name(&self) -> &'static str { self.type_name } fn as_any(&self) -> &dyn Any { self } fn into_rc_any(self: Rc) -> Rc { self } fn push_value(&self, value: Value) { if let Some(v) = T::from_value(value) { self.data.borrow_mut().push(v); } } fn as_series(&self) -> Option<&dyn Series> { Some(self) } } impl Series for ScalarSeries { fn get_item(&self, index: usize) -> Option { self.data.borrow().get(index).map(|&v| v.to_value()) } fn len(&self) -> usize { self.data.borrow().len() } fn total_count(&self) -> u64 { self.data.borrow().total_count() } } // ============================================================================ // 2b. Fallback Series for non-scalar types // ============================================================================ /// A generic series for all non-scalar types (Strings, Tuples, etc.) or mixed data. #[derive(Clone)] pub struct ValueSeries { pub data: std::cell::RefCell>, } impl ValueSeries { pub fn new(lookback_limit: usize) -> Self { Self { data: std::cell::RefCell::new(RingBuffer::new(lookback_limit)), } } } impl Debug for ValueSeries { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "ValueSeries[len: {}]", self.data.borrow().len()) } } impl Object for ValueSeries { fn type_name(&self) -> &'static str { "ValueSeries" } fn as_any(&self) -> &dyn Any { self } fn into_rc_any(self: Rc) -> Rc { self } fn push_value(&self, value: Value) { self.data.borrow_mut().push(value); } fn as_series(&self) -> Option<&dyn Series> { Some(self) } } impl Series for ValueSeries { fn get_item(&self, index: usize) -> Option { self.data.borrow().get(index).cloned() } fn len(&self) -> usize { self.data.borrow().len() } fn total_count(&self) -> u64 { self.data.borrow().total_count() } } // ============================================================================ // 3. RecordSeries (SoA - Struct of Arrays) // ============================================================================ /// Marker supertrait combining `Object` and `Series` for RecordSeries field buffers. /// Ensures each field buffer can be stored as a typed series AND used as a dynamic object /// (including `push_value` via the `Object` supertrait). pub trait SeriesMember: Object + Series {} impl SeriesMember for ScalarSeries {} impl SeriesMember for ScalarSeries {} impl SeriesMember for ScalarSeries {} impl SeriesMember for ValueSeries {} /// The "Struct of Arrays" implementation for Records (e.g., Ticks). /// Instead of holding a large array of Records (AoS), this series /// splits the record and stores each field in a separate (parallel) series. #[derive(Clone)] pub struct RecordSeries { layout: Arc, /// Each field of the record gets its own series (SoA). /// We use Rc so we can iterate fields individually, /// but also pass them as 0-Copy references to indicators! fields: Vec>>, } impl RecordSeries { /// Creates a new RecordSeries matching the given layout. pub fn new(layout: Arc, lookback_limit: usize) -> Self { let mut fields: Vec>> = Vec::with_capacity(layout.fields.len()); for (_, static_type) in &layout.fields { // Automatically select the optimal storage representation based on the field's static type. let member: Rc> = match static_type { StaticType::Float => Rc::new(std::cell::RefCell::new(ScalarSeries::::new( "FloatSeries", lookback_limit, ))), StaticType::Int | StaticType::DateTime => Rc::new(std::cell::RefCell::new( ScalarSeries::::new("IntSeries", lookback_limit), )), StaticType::Bool => Rc::new(std::cell::RefCell::new(ScalarSeries::::new( "BoolSeries", lookback_limit, ))), // Fallback for everything else (Text, Lists, nested Records without SoA, etc.) _ => Rc::new(std::cell::RefCell::new(ValueSeries::new(lookback_limit))), }; fields.push(member); } Self { layout, fields } } /// The magical 0-Copy Field Mapper! pub fn field(&self, key: Keyword) -> Option>> { self.layout .index_of(key) .map(|idx| self.fields[idx].clone()) } /// Dynamically reconstructs the full Record at the given lookback index. pub fn get_record(&self, index: usize) -> Option { if self.fields.is_empty() { return None; } let mut vals = Vec::with_capacity(self.fields.len()); for field in &self.fields { match field.borrow().get_item(index) { Some(v) => vals.push(v), None => return None, // Out of bounds or incomplete } } Some(Value::Record(self.layout.clone(), Rc::new(vals))) } } impl Debug for RecordSeries { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, "RecordSeries[fields: {}, len: {}]", self.fields.len(), Series::len(self) ) } } impl Object for RecordSeries { fn type_name(&self) -> &'static str { "RecordSeries" } fn as_any(&self) -> &dyn Any { self } fn into_rc_any(self: Rc) -> Rc { self } fn push_value(&self, value: Value) { if let Value::Record(_, values) = value { for (i, v) in values.iter().enumerate() { if let Some(f) = self.fields.get(i) { f.borrow().push_value(v.clone()); } } } else { panic!("push to RecordSeries expects a record"); } } fn as_series(&self) -> Option<&dyn Series> { Some(self) } } impl Series for RecordSeries { fn get_item(&self, index: usize) -> Option { self.get_record(index) } fn len(&self) -> usize { self.fields.first().map(|f| f.borrow().len()).unwrap_or(0) } fn total_count(&self) -> u64 { self.fields .first() .map(|f| f.borrow().total_count()) .unwrap_or(0) } } // ============================================================================ // 4. Series View (Safe, 0-Copy access bridge for the VM) // ============================================================================ #[derive(Clone)] pub struct SeriesView { pub inner: Rc>, pub field_name: Keyword, } impl SeriesView { pub fn new(inner: Rc>, field_name: Keyword) -> Self { Self { inner, field_name } } } impl Debug for SeriesView { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, "SeriesView[field: {}, len: {}]", self.field_name.name(), Series::len(self) ) } } impl Object for SeriesView { fn type_name(&self) -> &'static str { "SeriesView" } fn as_any(&self) -> &dyn Any { self } fn into_rc_any(self: Rc) -> Rc { self } fn as_series(&self) -> Option<&dyn Series> { Some(self) } } impl Series for SeriesView { fn get_item(&self, index: usize) -> Option { self.inner.borrow().get_item(index) } fn len(&self) -> usize { self.inner.borrow().len() } fn total_count(&self) -> u64 { self.inner.borrow().total_count() } } // ============================================================================ // 4b. Shared Series (Read-Only Reactive Storage) // ============================================================================ #[derive(Clone)] pub struct SharedSeries { pub buffer: Rc>>, pub type_name: &'static str, pub converter: fn(T) -> Value, } impl Debug for SharedSeries { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, "SharedSeries<{}>[len: {}]", self.type_name, self.buffer.borrow().len() ) } } impl Object for SharedSeries { fn type_name(&self) -> &'static str { self.type_name } fn as_any(&self) -> &dyn Any { self } fn into_rc_any(self: Rc) -> Rc { self } fn as_series(&self) -> Option<&dyn Series> { Some(self) } } impl Series for SharedSeries { fn get_item(&self, index: usize) -> Option { self.buffer .borrow() .get(index) .map(|&v| (self.converter)(v)) } fn len(&self) -> usize { self.buffer.borrow().len() } fn total_count(&self) -> u64 { self.buffer.borrow().total_count() } } #[derive(Clone)] pub struct SharedValueSeries { pub buffer: Rc>>, } impl Debug for SharedValueSeries { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "SharedValueSeries[len: {}]", self.buffer.borrow().len()) } } impl Object for SharedValueSeries { fn type_name(&self) -> &'static str { "SharedValueSeries" } fn as_any(&self) -> &dyn Any { self } fn into_rc_any(self: Rc) -> Rc { self } fn as_series(&self) -> Option<&dyn Series> { Some(self) } } impl Series for SharedValueSeries { fn get_item(&self, index: usize) -> Option { self.buffer.borrow().get(index).cloned() } fn len(&self) -> usize { self.buffer.borrow().len() } fn total_count(&self) -> u64 { self.buffer.borrow().total_count() } } #[derive(Clone)] pub struct SharedRecordSeries { pub layout: Arc, pub field_buffers: Vec>>, } impl Debug for SharedRecordSeries { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, "SharedRecordSeries[fields: {}, len: {}]", self.field_buffers.len(), Series::len(self) ) } } impl Object for SharedRecordSeries { fn type_name(&self) -> &'static str { "SharedRecordSeries" } fn as_any(&self) -> &dyn Any { self } fn into_rc_any(self: Rc) -> Rc { self } fn as_series(&self) -> Option<&dyn Series> { Some(self) } } impl Series for SharedRecordSeries { fn get_item(&self, index: usize) -> Option { if self.field_buffers.is_empty() { return None; } let mut vals = Vec::with_capacity(self.field_buffers.len()); for buffer in &self.field_buffers { match buffer.borrow().get_item(index) { Some(v) => vals.push(v), None => return None, } } Some(Value::Record(self.layout.clone(), Rc::new(vals))) } fn len(&self) -> usize { self.field_buffers .first() .map(|f| f.borrow().len()) .unwrap_or(0) } fn total_count(&self) -> u64 { self.field_buffers .first() .map(|f| f.borrow().total_count()) .unwrap_or(0) } } // ============================================================================ // 5. Script Integration (RTL Registration) // ============================================================================ pub fn register(env: &Environment) { // (series lookback_limit template_or_type_keyword) -> Series env.register_native_fn( "series", StaticType::Function(Box::new(Signature { params: StaticType::Tuple(vec![StaticType::Int, StaticType::Any]), ret: StaticType::Any, })), Purity::Impure, |args: &[Value]| { let lookback_limit = args[0].as_int().unwrap_or(0) as usize; let arg = &args[1]; match arg { Value::Keyword(k) => { match k.name().to_lowercase().as_str() { "float" => Value::Object(Rc::new(ScalarSeries::::new("FloatSeries", lookback_limit))), "int" | "datetime" => Value::Object(Rc::new(ScalarSeries::::new("IntSeries", lookback_limit))), "bool" => Value::Object(Rc::new(ScalarSeries::::new("BoolSeries", lookback_limit))), "text" => Value::Object(Rc::new(ValueSeries::new(lookback_limit))), _ => panic!("Unknown or unsupported series type keyword: :{}", k.name()), } } Value::Record(schema_layout, schema_values) => { // Interpret the record as a schema: { :field :type_keyword } let mut fields = Vec::with_capacity(schema_layout.fields.len()); for (i, (name, _)) in schema_layout.fields.iter().enumerate() { let type_keyword = match &schema_values[i] { Value::Keyword(tk) => tk.name().to_lowercase(), _ => panic!("Field :{} in series schema must be a type keyword (e.g., :float, :int)", name.name()), }; let ty = match type_keyword.as_str() { "float" => StaticType::Float, "int" => StaticType::Int, "bool" => StaticType::Bool, "datetime" => StaticType::DateTime, "text" => StaticType::Text, _ => panic!("Unsupported type keyword :{} for series field :{}", type_keyword, name.name()), }; fields.push((*name, ty)); } let final_layout = RecordLayout::get_or_create(fields); Value::Object(Rc::new(RecordSeries::new(final_layout, lookback_limit))) } _ => panic!("series expects a lookback_limit and a type keyword (:float, :int...) or a schema record (e.g., {{:price :float}})"), } }, ); // (push series value) -> Void env.register_native_fn( "push", StaticType::Function(Box::new(Signature { params: StaticType::Tuple(vec![StaticType::Any, StaticType::Any]), ret: StaticType::Void, })), Purity::Impure, |args: &[Value]| { let (series_val, val) = (&args[0], &args[1]); if let Value::Object(obj) = series_val { obj.push_value(val.clone()); return Value::Void; } panic!("push expects a series object as the first argument") }, ); // (len series) -> Int env.register_native_fn( "len", StaticType::Function(Box::new(Signature { params: StaticType::Tuple(vec![StaticType::Any]), ret: StaticType::Int, })), Purity::Pure, |args: &[Value]| { if let Value::Object(obj) = &args[0] && let Some(s) = obj.as_series() { return Value::Int(s.len() as i64); } Value::Int(0) }, ); }