Add Series RTL and SeriesView
Introduces a new RTL module for handling series data, enabling efficient storage and retrieval of time-series data in a Struct-of-Arrays (SoA) format. Includes `SeriesView`, a zero-copy wrapper that provides a fast, read-only interface to individual columns within a `RecordSeries`, optimizing performance for data access in the VM. Registers new native functions `create-series` and `push-series` for managing series data.
This commit is contained in:
Michael Schimmel
@@ -0,0 +1,15 @@
|
||||
;; Output: 26.7
|
||||
;; Benchmark: 998ns
|
||||
;; Benchmark-Repeat: 2047
|
||||
(do
|
||||
(def template {:price 10.0 :volume 100})
|
||||
(def my_ticks (create-series template))
|
||||
|
||||
(push-series my_ticks {:price 10.5 :volume 100})
|
||||
(push-series my_ticks {:price 11.2 :volume 200})
|
||||
(push-series my_ticks {:price 15.5 :volume 300})
|
||||
|
||||
(def prices (.price my_ticks))
|
||||
|
||||
(+ (prices 0) (prices 1))
|
||||
)
|
||||
@@ -11,4 +11,5 @@ pub fn register(env: &Environment) {
|
||||
core::register(env);
|
||||
datetime::register(env);
|
||||
math::register(env);
|
||||
series::register(env);
|
||||
}
|
||||
+156
-4
@@ -161,12 +161,56 @@ impl Object for ValueSeries {
|
||||
/// An interface allowing iteration over fields of a RecordSeries independently
|
||||
/// of the exact type (Type Erasure for member arrays).
|
||||
pub trait SeriesMember: Object {
|
||||
// We could add type-safe `push_any(&mut self, Value)` here later,
|
||||
// if the fallback path of the VM needs to insert values dynamically.
|
||||
/// Gets the value at the specified lookback index as a dynamic Value.
|
||||
fn get_value(&self, index: usize) -> Option<Value>;
|
||||
|
||||
/// Pushes a dynamic Value into the series, performing the necessary downcast.
|
||||
fn push_value(&mut self, value: Value, limit: Option<usize>);
|
||||
}
|
||||
|
||||
impl<T: ScalarValue> SeriesMember for ScalarSeries<T> {}
|
||||
impl SeriesMember for ValueSeries {}
|
||||
impl SeriesMember for ScalarSeries<f64> {
|
||||
fn get_value(&self, index: usize) -> Option<Value> {
|
||||
self.data.get(index).map(|&v| Value::Float(v))
|
||||
}
|
||||
fn push_value(&mut self, value: Value, limit: Option<usize>) {
|
||||
if let Value::Float(v) = value {
|
||||
self.data.push(v, limit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SeriesMember for ScalarSeries<i64> {
|
||||
fn get_value(&self, index: usize) -> Option<Value> {
|
||||
self.data.get(index).map(|&v| Value::Int(v))
|
||||
}
|
||||
fn push_value(&mut self, value: Value, limit: Option<usize>) {
|
||||
if let Value::Int(v) = value {
|
||||
self.data.push(v, limit);
|
||||
} else if let Value::DateTime(v) = value {
|
||||
self.data.push(v, limit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SeriesMember for ScalarSeries<bool> {
|
||||
fn get_value(&self, index: usize) -> Option<Value> {
|
||||
self.data.get(index).map(|&v| Value::Bool(v))
|
||||
}
|
||||
fn push_value(&mut self, value: Value, limit: Option<usize>) {
|
||||
if let Value::Bool(v) = value {
|
||||
self.data.push(v, limit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SeriesMember for ValueSeries {
|
||||
fn get_value(&self, index: usize) -> Option<Value> {
|
||||
self.data.get(index).cloned()
|
||||
}
|
||||
fn push_value(&mut self, value: Value, limit: Option<usize>) {
|
||||
self.data.push(value, limit);
|
||||
}
|
||||
}
|
||||
|
||||
/// The "Struct of Arrays" implementation for Records (e.g., Ticks).
|
||||
/// Instead of holding a large array of Records (AoS), this series
|
||||
@@ -223,3 +267,111 @@ impl Object for RecordSeries {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// 4. Series View (Safe, 0-Copy access bridge for the VM)
|
||||
// ============================================================================
|
||||
|
||||
/// A lightweight, 0-copy wrapper around a specific column (field) of a RecordSeries.
|
||||
///
|
||||
/// Why is this needed?
|
||||
/// In a SoA (Struct of Arrays) architecture, columns like `open` or `close`
|
||||
/// are stored as `Rc<RefCell<dyn SeriesMember>>` inside the `RecordSeries`.
|
||||
/// The `RefCell` is crucial for runtime mutability (so the engine can push new ticks).
|
||||
/// However, the VM's `Value::Object` expects a pure `Rc<dyn Object>`.
|
||||
///
|
||||
/// `SeriesView` bridges this gap safely: It holds the shared reference and implements `Object` itself,
|
||||
/// acting as a fast, read-only "window" into the underlying column data without copying any arrays.
|
||||
#[derive(Clone)]
|
||||
pub struct SeriesView {
|
||||
pub inner: Rc<std::cell::RefCell<dyn SeriesMember>>,
|
||||
pub field_name: Keyword,
|
||||
}
|
||||
|
||||
impl SeriesView {
|
||||
pub fn new(inner: Rc<std::cell::RefCell<dyn SeriesMember>>, field_name: Keyword) -> Self {
|
||||
Self { inner, field_name }
|
||||
}
|
||||
}
|
||||
|
||||
impl Debug for SeriesView {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "SeriesView[field: {}]", self.field_name.name())
|
||||
}
|
||||
}
|
||||
|
||||
impl Object for SeriesView {
|
||||
fn type_name(&self) -> &'static str {
|
||||
"SeriesView"
|
||||
}
|
||||
|
||||
// This allows the VM or specialized extractors to downcast the View back to its concrete type if needed.
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// 5. Script Integration (RTL Registration)
|
||||
// ============================================================================
|
||||
|
||||
use crate::ast::environment::Environment;
|
||||
use crate::ast::types::{Purity, Signature};
|
||||
|
||||
pub fn register(env: &Environment) {
|
||||
// (create-series layout_record) -> RecordSeries
|
||||
// Extracts the layout from a sample record and creates an empty SoA RecordSeries.
|
||||
env.register_native_fn(
|
||||
"create-series",
|
||||
StaticType::Function(Box::new(Signature {
|
||||
params: StaticType::Tuple(vec![StaticType::Any]), // In reality: Record
|
||||
ret: StaticType::Any, // In reality: Object(RecordSeries)
|
||||
})),
|
||||
Purity::Impure,
|
||||
|args: std::vec::Vec<Value>| {
|
||||
if args.len() != 1 {
|
||||
panic!("create-series expects exactly 1 argument (a record)");
|
||||
}
|
||||
if let Value::Record(layout, _) = &args[0] {
|
||||
let series = RecordSeries::new(layout.clone());
|
||||
Value::Object(Rc::new(series))
|
||||
} else {
|
||||
panic!("create-series expects a record to extract the layout from")
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// (push-series series record) -> Void
|
||||
// Pushes the values of a record into the parallel arrays of the RecordSeries.
|
||||
env.register_native_fn(
|
||||
"push-series",
|
||||
StaticType::Function(Box::new(Signature {
|
||||
params: StaticType::Tuple(vec![StaticType::Any, StaticType::Any]),
|
||||
ret: StaticType::Void,
|
||||
})),
|
||||
Purity::Impure,
|
||||
|args: std::vec::Vec<Value>| {
|
||||
if args.len() != 2 {
|
||||
panic!("push-series expects exactly 2 arguments (series, record)");
|
||||
}
|
||||
let (series_val, record_val) = (&args[0], &args[1]);
|
||||
|
||||
if let Value::Object(obj) = series_val {
|
||||
if let Some(record_series) = obj.as_any().downcast_ref::<RecordSeries>() {
|
||||
if let Value::Record(_, values) = record_val {
|
||||
// Push each value into its corresponding column (SoA push)
|
||||
for (i, field_val) in values.iter().enumerate() {
|
||||
if let Some(field_series) = record_series.fields.get(i) {
|
||||
field_series.borrow_mut().push_value(field_val.clone(), None);
|
||||
}
|
||||
}
|
||||
return Value::Void;
|
||||
} else {
|
||||
panic!("push-series expects a record as the second argument");
|
||||
}
|
||||
}
|
||||
}
|
||||
panic!("push-series expects a RecordSeries as the first argument")
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -284,6 +284,7 @@ pub enum StaticType {
|
||||
Text,
|
||||
Keyword,
|
||||
List(Box<StaticType>), // Legacy / Dynamic list
|
||||
Series(Box<StaticType>), // Time series of a specific type
|
||||
Tuple(Vec<StaticType>), // Heterogeneous fixed-size
|
||||
Vector(Box<StaticType>, usize), // Homogeneous fixed-size
|
||||
Matrix(Box<StaticType>, Vec<usize>), // Multi-dimensional homogeneous
|
||||
@@ -306,6 +307,7 @@ impl fmt::Display for StaticType {
|
||||
StaticType::Text => write!(f, "text"),
|
||||
StaticType::Keyword => write!(f, "keyword"),
|
||||
StaticType::List(inner) => write!(f, "[{}]", inner),
|
||||
StaticType::Series(inner) => write!(f, "series<{}>", inner),
|
||||
StaticType::Tuple(elements) => {
|
||||
write!(f, "[")?;
|
||||
for (i, el) in elements.iter().enumerate() {
|
||||
@@ -391,6 +393,10 @@ impl StaticType {
|
||||
(StaticType::Record(a), StaticType::Record(b)) => {
|
||||
std::sync::Arc::ptr_eq(a, b)
|
||||
}
|
||||
// Series are assignable if their inner types are assignable
|
||||
(StaticType::Series(inner_a), StaticType::Series(inner_b)) => {
|
||||
inner_a.is_assignable_from(inner_b)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
+82
-16
@@ -301,14 +301,51 @@ impl VM {
|
||||
|
||||
BoundKind::GetField { rec, field } => {
|
||||
let rec_val = self.eval_internal(obs, rec)?;
|
||||
if let Value::Record(layout, values) = rec_val {
|
||||
|
||||
// In Rust, pattern matching (`match`) is the idiomatic way to handle variants safely.
|
||||
// Previously, this only handled `Value::Record`. Now, we handle objects (like `RecordSeries`) polymorphically.
|
||||
match rec_val {
|
||||
// Case 1: The classic Record.
|
||||
// This is a struct-like tuple containing an Arc<RecordLayout> and a Vec<Value>.
|
||||
Value::Record(layout, values) => {
|
||||
if let Some(idx) = layout.index_of(*field) {
|
||||
Ok(values[idx].clone())
|
||||
} else {
|
||||
Err(format!("Record does not have field :{}", field.name()))
|
||||
}
|
||||
}
|
||||
|
||||
// Case 2: A dynamic Object (our SoA / Struct-of-Arrays optimization).
|
||||
// `Value::Object` holds an `Rc<dyn Object>` - a reference-counted trait object (type-erased).
|
||||
Value::Object(obj) => {
|
||||
// 1. We get the raw `&dyn Any` reference (Rust's standard mechanism for runtime type reflection).
|
||||
let any_ptr = obj.as_any();
|
||||
|
||||
// 2. Downcast! We check at runtime if the pointer actually points to a `RecordSeries`.
|
||||
// `downcast_ref` is very fast (essentially an O(1) type ID comparison under the hood).
|
||||
if let Some(record_series) = any_ptr.downcast_ref::<crate::ast::rtl::series::RecordSeries>() {
|
||||
|
||||
// 3. We call our highly performant 0-copy method on the series.
|
||||
// It returns an `Rc<RefCell<dyn SeriesMember>>`, which is a shared pointer
|
||||
// to the concrete column array (e.g., a `ScalarSeries<f64>`).
|
||||
if let Some(field_series) = record_series.field(*field) {
|
||||
// 4. We wrap this RefCell in our `SeriesView` struct.
|
||||
// The `SeriesView` acts as a pure `Object` for the VM, holding the reference.
|
||||
// CRITICAL: No array elements are copied here! This is pure, fast pointer juggling.
|
||||
// This single operation turns a SoA `RecordSeries` into a high-speed `FloatSeries` view.
|
||||
let view = crate::ast::rtl::series::SeriesView::new(field_series, *field);
|
||||
return Ok(Value::Object(std::rc::Rc::new(view)));
|
||||
} else {
|
||||
Err(format!("Attempt to access field on non-record: {}", rec_val))
|
||||
return Err(format!("RecordSeries does not have field :{}", field.name()));
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback if it's another type of object that is not a RecordSeries.
|
||||
Err(format!("Attempt to access field on non-record object: {}", obj.type_name()))
|
||||
}
|
||||
|
||||
// Error handling for primitives (Int, Float, etc.).
|
||||
_ => Err(format!("Attempt to access field on non-record: {}", rec_val)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -401,15 +438,21 @@ impl VM {
|
||||
} else {
|
||||
return Err(format!("Record does not have field :{}", k.name()));
|
||||
}
|
||||
} else if let Value::Object(obj) = rec {
|
||||
// Polymorphic Field Access: Allow `.field` on a RecordSeries
|
||||
if let Some(rs) = obj.as_any().downcast_ref::<crate::ast::rtl::series::RecordSeries>() {
|
||||
if let Some(field_series) = rs.field(k) {
|
||||
let view = crate::ast::rtl::series::SeriesView::new(field_series, k);
|
||||
return Ok(Value::Object(std::rc::Rc::new(view)));
|
||||
} else {
|
||||
return Err(format!(
|
||||
"Field accessor .{} expects a record, got {}",
|
||||
k.name(),
|
||||
rec
|
||||
));
|
||||
return Err(format!("RecordSeries does not have field :{}", k.name()));
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
return Err(format!("Field accessor .{} expects a record or RecordSeries, got {}", k.name(), obj.type_name()));
|
||||
} else {
|
||||
return Err(format!("Field accessor .{} expects a record or RecordSeries, got {}", k.name(), rec));
|
||||
}
|
||||
} _ => {
|
||||
return Err(format!(
|
||||
"Tail call target is not a function: {}",
|
||||
func_val
|
||||
@@ -436,15 +479,21 @@ impl VM {
|
||||
} else {
|
||||
break Err(format!("Record does not have field :{}", k.name()));
|
||||
}
|
||||
} else if let Value::Object(obj) = rec {
|
||||
// Polymorphic Field Access: Allow `.field` on a RecordSeries
|
||||
if let Some(rs) = obj.as_any().downcast_ref::<crate::ast::rtl::series::RecordSeries>() {
|
||||
if let Some(field_series) = rs.field(k) {
|
||||
let view = crate::ast::rtl::series::SeriesView::new(field_series, k);
|
||||
break Ok(Value::Object(std::rc::Rc::new(view)));
|
||||
} else {
|
||||
break Err(format!(
|
||||
"Field accessor .{} expects a record, got {}",
|
||||
k.name(),
|
||||
rec
|
||||
));
|
||||
break Err(format!("RecordSeries does not have field :{}", k.name()));
|
||||
}
|
||||
}
|
||||
Value::Object(obj) => {
|
||||
break Err(format!("Field accessor .{} expects a record or RecordSeries, got {}", k.name(), obj.type_name()));
|
||||
} else {
|
||||
break Err(format!("Field accessor .{} expects a record or RecordSeries, got {}", k.name(), rec));
|
||||
}
|
||||
} Value::Object(obj) => {
|
||||
if let Some(closure) = obj.as_any().downcast_ref::<Closure>() {
|
||||
let old_stack_top = self.stack.len();
|
||||
self.frames.push(CallFrame {
|
||||
@@ -480,11 +529,28 @@ impl VM {
|
||||
}
|
||||
res => break res,
|
||||
}
|
||||
} else if let Some(view) = obj.as_any().downcast_ref::<crate::ast::rtl::series::SeriesView>() {
|
||||
// Support for `(prices 0)` or `prices[0]` - reading from the 0-copy view!
|
||||
if arg_vals.len() != 1 {
|
||||
break Err("Series indexer expects exactly 1 argument (the lookback index)".to_string());
|
||||
}
|
||||
if let Value::Int(idx) = arg_vals[0] {
|
||||
if idx < 0 {
|
||||
break Err("Series lookback index cannot be negative".to_string());
|
||||
}
|
||||
if let Some(val) = view.inner.borrow().get_value(idx as usize) {
|
||||
break Ok(val);
|
||||
} else {
|
||||
break Err(format!("Object is not a closure: {}", obj.type_name()));
|
||||
// Out of bounds / not enough data yet
|
||||
break Ok(Value::Void);
|
||||
}
|
||||
} else {
|
||||
break Err("Series index must be an integer".to_string());
|
||||
}
|
||||
_ => break Err(format!("Attempt to call non-function: {}", func_val)),
|
||||
} else {
|
||||
break Err(format!("Object is not callable: {}", obj.type_name()));
|
||||
}
|
||||
} _ => break Err(format!("Attempt to call non-function: {}", func_val)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user