From b612df6e104305501b1c19ff8d3df2537d4ef56f Mon Sep 17 00:00:00 2001 From: Michael Schimmel Date: Sat, 28 Feb 2026 21:47:14 +0100 Subject: [PATCH] Add Series RTL and `SeriesView` Introduces a new RTL module for handling series data, enabling efficient storage and retrieval of time-series data in a Struct-of-Arrays (SoA) format. Includes `SeriesView`, a zero-copy wrapper that provides a fast, read-only interface to individual columns within a `RecordSeries`, optimizing performance for data access in the VM. Registers new native functions `create-series` and `push-series` for managing series data. --- examples/soa_series.myc | 15 ++++ src/ast/rtl/mod.rs | 1 + src/ast/rtl/series.rs | 160 +++++++++++++++++++++++++++++++++++++++- src/ast/types.rs | 6 ++ src/ast/vm.rs | 114 ++++++++++++++++++++++------ 5 files changed, 268 insertions(+), 28 deletions(-) create mode 100644 examples/soa_series.myc diff --git a/examples/soa_series.myc b/examples/soa_series.myc new file mode 100644 index 0000000..3c94d4a --- /dev/null +++ b/examples/soa_series.myc @@ -0,0 +1,15 @@ +;; Output: 26.7 +;; Benchmark: 998ns +;; Benchmark-Repeat: 2047 +(do + (def template {:price 10.0 :volume 100}) + (def my_ticks (create-series template)) + + (push-series my_ticks {:price 10.5 :volume 100}) + (push-series my_ticks {:price 11.2 :volume 200}) + (push-series my_ticks {:price 15.5 :volume 300}) + + (def prices (.price my_ticks)) + + (+ (prices 0) (prices 1)) +) \ No newline at end of file diff --git a/src/ast/rtl/mod.rs b/src/ast/rtl/mod.rs index 91b6d64..7eaaed1 100644 --- a/src/ast/rtl/mod.rs +++ b/src/ast/rtl/mod.rs @@ -11,4 +11,5 @@ pub fn register(env: &Environment) { core::register(env); datetime::register(env); math::register(env); + series::register(env); } \ No newline at end of file diff --git a/src/ast/rtl/series.rs b/src/ast/rtl/series.rs index 0841415..d63efdb 100644 --- a/src/ast/rtl/series.rs +++ b/src/ast/rtl/series.rs @@ -161,12 +161,56 @@ impl Object for ValueSeries { /// An interface allowing iteration over fields of a RecordSeries independently /// of the exact type (Type Erasure for member arrays). pub trait SeriesMember: Object { - // We could add type-safe `push_any(&mut self, Value)` here later, - // if the fallback path of the VM needs to insert values dynamically. + /// Gets the value at the specified lookback index as a dynamic Value. + fn get_value(&self, index: usize) -> Option; + + /// Pushes a dynamic Value into the series, performing the necessary downcast. + fn push_value(&mut self, value: Value, limit: Option); } -impl SeriesMember for ScalarSeries {} -impl SeriesMember for ValueSeries {} +impl SeriesMember for ScalarSeries { + fn get_value(&self, index: usize) -> Option { + self.data.get(index).map(|&v| Value::Float(v)) + } + fn push_value(&mut self, value: Value, limit: Option) { + if let Value::Float(v) = value { + self.data.push(v, limit); + } + } +} + +impl SeriesMember for ScalarSeries { + fn get_value(&self, index: usize) -> Option { + self.data.get(index).map(|&v| Value::Int(v)) + } + fn push_value(&mut self, value: Value, limit: Option) { + if let Value::Int(v) = value { + self.data.push(v, limit); + } else if let Value::DateTime(v) = value { + self.data.push(v, limit); + } + } +} + +impl SeriesMember for ScalarSeries { + fn get_value(&self, index: usize) -> Option { + self.data.get(index).map(|&v| Value::Bool(v)) + } + fn push_value(&mut self, value: Value, limit: Option) { + if let Value::Bool(v) = value { + self.data.push(v, limit); + } + } +} + +impl SeriesMember for ValueSeries { + fn get_value(&self, index: usize) -> Option { + self.data.get(index).cloned() + } + fn push_value(&mut self, value: Value, limit: Option) { + self.data.push(value, limit); + } +} /// The "Struct of Arrays" implementation for Records (e.g., Ticks). /// Instead of holding a large array of Records (AoS), this series @@ -223,3 +267,111 @@ impl Object for RecordSeries { self } } + +// ============================================================================ +// 4. Series View (Safe, 0-Copy access bridge for the VM) +// ============================================================================ + +/// A lightweight, 0-copy wrapper around a specific column (field) of a RecordSeries. +/// +/// Why is this needed? +/// In a SoA (Struct of Arrays) architecture, columns like `open` or `close` +/// are stored as `Rc>` inside the `RecordSeries`. +/// The `RefCell` is crucial for runtime mutability (so the engine can push new ticks). +/// However, the VM's `Value::Object` expects a pure `Rc`. +/// +/// `SeriesView` bridges this gap safely: It holds the shared reference and implements `Object` itself, +/// acting as a fast, read-only "window" into the underlying column data without copying any arrays. +#[derive(Clone)] +pub struct SeriesView { + pub inner: Rc>, + pub field_name: Keyword, +} + +impl SeriesView { + pub fn new(inner: Rc>, field_name: Keyword) -> Self { + Self { inner, field_name } + } +} + +impl Debug for SeriesView { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "SeriesView[field: {}]", self.field_name.name()) + } +} + +impl Object for SeriesView { + fn type_name(&self) -> &'static str { + "SeriesView" + } + + // This allows the VM or specialized extractors to downcast the View back to its concrete type if needed. + fn as_any(&self) -> &dyn Any { + self + } +} + +// ============================================================================ +// 5. Script Integration (RTL Registration) +// ============================================================================ + +use crate::ast::environment::Environment; +use crate::ast::types::{Purity, Signature}; + +pub fn register(env: &Environment) { + // (create-series layout_record) -> RecordSeries + // Extracts the layout from a sample record and creates an empty SoA RecordSeries. + env.register_native_fn( + "create-series", + StaticType::Function(Box::new(Signature { + params: StaticType::Tuple(vec![StaticType::Any]), // In reality: Record + ret: StaticType::Any, // In reality: Object(RecordSeries) + })), + Purity::Impure, + |args: std::vec::Vec| { + if args.len() != 1 { + panic!("create-series expects exactly 1 argument (a record)"); + } + if let Value::Record(layout, _) = &args[0] { + let series = RecordSeries::new(layout.clone()); + Value::Object(Rc::new(series)) + } else { + panic!("create-series expects a record to extract the layout from") + } + }, + ); + + // (push-series series record) -> Void + // Pushes the values of a record into the parallel arrays of the RecordSeries. + env.register_native_fn( + "push-series", + StaticType::Function(Box::new(Signature { + params: StaticType::Tuple(vec![StaticType::Any, StaticType::Any]), + ret: StaticType::Void, + })), + Purity::Impure, + |args: std::vec::Vec| { + if args.len() != 2 { + panic!("push-series expects exactly 2 arguments (series, record)"); + } + let (series_val, record_val) = (&args[0], &args[1]); + + if let Value::Object(obj) = series_val { + if let Some(record_series) = obj.as_any().downcast_ref::() { + if let Value::Record(_, values) = record_val { + // Push each value into its corresponding column (SoA push) + for (i, field_val) in values.iter().enumerate() { + if let Some(field_series) = record_series.fields.get(i) { + field_series.borrow_mut().push_value(field_val.clone(), None); + } + } + return Value::Void; + } else { + panic!("push-series expects a record as the second argument"); + } + } + } + panic!("push-series expects a RecordSeries as the first argument") + }, + ); +} diff --git a/src/ast/types.rs b/src/ast/types.rs index 9e9c61f..d810b7b 100644 --- a/src/ast/types.rs +++ b/src/ast/types.rs @@ -284,6 +284,7 @@ pub enum StaticType { Text, Keyword, List(Box), // Legacy / Dynamic list + Series(Box), // Time series of a specific type Tuple(Vec), // Heterogeneous fixed-size Vector(Box, usize), // Homogeneous fixed-size Matrix(Box, Vec), // Multi-dimensional homogeneous @@ -306,6 +307,7 @@ impl fmt::Display for StaticType { StaticType::Text => write!(f, "text"), StaticType::Keyword => write!(f, "keyword"), StaticType::List(inner) => write!(f, "[{}]", inner), + StaticType::Series(inner) => write!(f, "series<{}>", inner), StaticType::Tuple(elements) => { write!(f, "[")?; for (i, el) in elements.iter().enumerate() { @@ -391,6 +393,10 @@ impl StaticType { (StaticType::Record(a), StaticType::Record(b)) => { std::sync::Arc::ptr_eq(a, b) } + // Series are assignable if their inner types are assignable + (StaticType::Series(inner_a), StaticType::Series(inner_b)) => { + inner_a.is_assignable_from(inner_b) + } _ => false, } } diff --git a/src/ast/vm.rs b/src/ast/vm.rs index c1b0a3c..a8ba5e8 100644 --- a/src/ast/vm.rs +++ b/src/ast/vm.rs @@ -301,14 +301,51 @@ impl VM { BoundKind::GetField { rec, field } => { let rec_val = self.eval_internal(obs, rec)?; - if let Value::Record(layout, values) = rec_val { - if let Some(idx) = layout.index_of(*field) { - Ok(values[idx].clone()) - } else { - Err(format!("Record does not have field :{}", field.name())) + + // In Rust, pattern matching (`match`) is the idiomatic way to handle variants safely. + // Previously, this only handled `Value::Record`. Now, we handle objects (like `RecordSeries`) polymorphically. + match rec_val { + // Case 1: The classic Record. + // This is a struct-like tuple containing an Arc and a Vec. + Value::Record(layout, values) => { + if let Some(idx) = layout.index_of(*field) { + Ok(values[idx].clone()) + } else { + Err(format!("Record does not have field :{}", field.name())) + } } - } else { - Err(format!("Attempt to access field on non-record: {}", rec_val)) + + // Case 2: A dynamic Object (our SoA / Struct-of-Arrays optimization). + // `Value::Object` holds an `Rc` - a reference-counted trait object (type-erased). + Value::Object(obj) => { + // 1. We get the raw `&dyn Any` reference (Rust's standard mechanism for runtime type reflection). + let any_ptr = obj.as_any(); + + // 2. Downcast! We check at runtime if the pointer actually points to a `RecordSeries`. + // `downcast_ref` is very fast (essentially an O(1) type ID comparison under the hood). + if let Some(record_series) = any_ptr.downcast_ref::() { + + // 3. We call our highly performant 0-copy method on the series. + // It returns an `Rc>`, which is a shared pointer + // to the concrete column array (e.g., a `ScalarSeries`). + if let Some(field_series) = record_series.field(*field) { + // 4. We wrap this RefCell in our `SeriesView` struct. + // The `SeriesView` acts as a pure `Object` for the VM, holding the reference. + // CRITICAL: No array elements are copied here! This is pure, fast pointer juggling. + // This single operation turns a SoA `RecordSeries` into a high-speed `FloatSeries` view. + let view = crate::ast::rtl::series::SeriesView::new(field_series, *field); + return Ok(Value::Object(std::rc::Rc::new(view))); + } else { + return Err(format!("RecordSeries does not have field :{}", field.name())); + } + } + + // Fallback if it's another type of object that is not a RecordSeries. + Err(format!("Attempt to access field on non-record object: {}", obj.type_name())) + } + + // Error handling for primitives (Int, Float, etc.). + _ => Err(format!("Attempt to access field on non-record: {}", rec_val)), } } @@ -401,15 +438,21 @@ impl VM { } else { return Err(format!("Record does not have field :{}", k.name())); } + } else if let Value::Object(obj) = rec { + // Polymorphic Field Access: Allow `.field` on a RecordSeries + if let Some(rs) = obj.as_any().downcast_ref::() { + if let Some(field_series) = rs.field(k) { + let view = crate::ast::rtl::series::SeriesView::new(field_series, k); + return Ok(Value::Object(std::rc::Rc::new(view))); + } else { + return Err(format!("RecordSeries does not have field :{}", k.name())); + } + } + return Err(format!("Field accessor .{} expects a record or RecordSeries, got {}", k.name(), obj.type_name())); } else { - return Err(format!( - "Field accessor .{} expects a record, got {}", - k.name(), - rec - )); + return Err(format!("Field accessor .{} expects a record or RecordSeries, got {}", k.name(), rec)); } - } - _ => { + } _ => { return Err(format!( "Tail call target is not a function: {}", func_val @@ -436,15 +479,21 @@ impl VM { } else { break Err(format!("Record does not have field :{}", k.name())); } + } else if let Value::Object(obj) = rec { + // Polymorphic Field Access: Allow `.field` on a RecordSeries + if let Some(rs) = obj.as_any().downcast_ref::() { + if let Some(field_series) = rs.field(k) { + let view = crate::ast::rtl::series::SeriesView::new(field_series, k); + break Ok(Value::Object(std::rc::Rc::new(view))); + } else { + break Err(format!("RecordSeries does not have field :{}", k.name())); + } + } + break Err(format!("Field accessor .{} expects a record or RecordSeries, got {}", k.name(), obj.type_name())); } else { - break Err(format!( - "Field accessor .{} expects a record, got {}", - k.name(), - rec - )); + break Err(format!("Field accessor .{} expects a record or RecordSeries, got {}", k.name(), rec)); } - } - Value::Object(obj) => { + } Value::Object(obj) => { if let Some(closure) = obj.as_any().downcast_ref::() { let old_stack_top = self.stack.len(); self.frames.push(CallFrame { @@ -480,11 +529,28 @@ impl VM { } res => break res, } + } else if let Some(view) = obj.as_any().downcast_ref::() { + // Support for `(prices 0)` or `prices[0]` - reading from the 0-copy view! + if arg_vals.len() != 1 { + break Err("Series indexer expects exactly 1 argument (the lookback index)".to_string()); + } + if let Value::Int(idx) = arg_vals[0] { + if idx < 0 { + break Err("Series lookback index cannot be negative".to_string()); + } + if let Some(val) = view.inner.borrow().get_value(idx as usize) { + break Ok(val); + } else { + // Out of bounds / not enough data yet + break Ok(Value::Void); + } + } else { + break Err("Series index must be an integer".to_string()); + } } else { - break Err(format!("Object is not a closure: {}", obj.type_name())); + break Err(format!("Object is not callable: {}", obj.type_name())); } - } - _ => break Err(format!("Attempt to call non-function: {}", func_val)), + } _ => break Err(format!("Attempt to call non-function: {}", func_val)), } } }