Michael Schimmel
595bcf09e5
The `PipelineNode` has been refactored into `StreamNode`. This commit updates the example files to reflect this change and also updates the `series` constructor to accept a lookback parameter. The `PipelineNode` was an artifact of a previous implementation and is no longer needed. The `StreamNode` is the appropriate type for representing the output of a pipe operation. The `series` function now requires a `lookback` argument to specify the maximum number of items to store. This makes the behavior of series more explicit and prevents accidental unbounded memory growth.
649 lines
20 KiB
Rust
649 lines
20 KiB
Rust
use std::any::Any;
|
|
use std::collections::VecDeque;
|
|
use std::fmt::{self, Debug};
|
|
use std::rc::Rc;
|
|
use std::sync::Arc;
|
|
|
|
use crate::ast::environment::Environment;
|
|
use crate::ast::types::{Keyword, Object, RecordLayout, Series, StaticType, Value};
|
|
use crate::ast::types::{Purity, Signature};
|
|
|
|
// ============================================================================
|
|
// 1. Core Storage Engine
|
|
// ============================================================================
|
|
|
|
/// A generic, efficient ring buffer for time series data.
|
|
/// (Replaces Delphi's TChunkArray for now with VecDeque for O(1) push/pop)
|
|
#[derive(Clone)]
|
|
pub struct RingBuffer<T> {
|
|
buffer: VecDeque<T>,
|
|
total_count: u64,
|
|
lookback_limit: usize,
|
|
}
|
|
|
|
impl<T> RingBuffer<T> {
|
|
pub fn new(lookback_limit: usize) -> Self {
|
|
// Pre-allocate up to 500 elements to minimize heap fragmentation
|
|
let initial_capacity = std::cmp::min(lookback_limit, 500);
|
|
Self {
|
|
buffer: VecDeque::with_capacity(initial_capacity),
|
|
total_count: 0,
|
|
lookback_limit,
|
|
}
|
|
}
|
|
|
|
/// Adds an element. Removes the oldest if the lookback limit is reached.
|
|
#[inline]
|
|
pub fn push(&mut self, item: T) {
|
|
self.buffer.push_back(item);
|
|
self.total_count += 1;
|
|
|
|
while self.buffer.len() > self.lookback_limit {
|
|
self.buffer.pop_front();
|
|
}
|
|
}
|
|
|
|
/// Access in "Financial Style": Index 0 is the newest element.
|
|
#[inline]
|
|
pub fn get(&self, index: usize) -> Option<&T> {
|
|
let len = self.buffer.len();
|
|
if index < len {
|
|
self.buffer.get(len - 1 - index)
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
#[inline]
|
|
pub fn total_count(&self) -> u64 {
|
|
self.total_count
|
|
}
|
|
|
|
#[inline]
|
|
pub fn len(&self) -> usize {
|
|
self.buffer.len()
|
|
}
|
|
|
|
pub fn is_empty(&self) -> bool {
|
|
self.buffer.is_empty()
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// 2. Trait & Primitives for Fast-Paths
|
|
// ============================================================================
|
|
|
|
/// Marker trait ensuring our fast arrays only store true, flat scalars
|
|
/// (No pointers/heaps like String or Record).
|
|
pub trait ScalarValue: Copy + Debug + 'static {
|
|
fn to_value(self) -> Value;
|
|
}
|
|
|
|
impl ScalarValue for f64 {
|
|
fn to_value(self) -> Value {
|
|
Value::Float(self)
|
|
}
|
|
}
|
|
impl ScalarValue for i64 {
|
|
fn to_value(self) -> Value {
|
|
Value::Int(self)
|
|
}
|
|
}
|
|
impl ScalarValue for bool {
|
|
fn to_value(self) -> Value {
|
|
Value::Bool(self)
|
|
}
|
|
}
|
|
|
|
/// A highly optimized, type-safe series for primitive values (Float, Int, Bool).
|
|
#[derive(Clone)]
|
|
pub struct ScalarSeries<T: ScalarValue> {
|
|
pub data: std::cell::RefCell<RingBuffer<T>>,
|
|
pub type_name: &'static str,
|
|
}
|
|
|
|
impl<T: ScalarValue> ScalarSeries<T> {
|
|
pub fn new(type_name: &'static str, lookback_limit: usize) -> Self {
|
|
Self {
|
|
data: std::cell::RefCell::new(RingBuffer::new(lookback_limit)),
|
|
type_name,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<T: ScalarValue> Debug for ScalarSeries<T> {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
write!(
|
|
f,
|
|
"ScalarSeries<{}>[len: {}]",
|
|
self.type_name,
|
|
self.data.borrow().len()
|
|
)
|
|
}
|
|
}
|
|
|
|
// Makes the series usable as a dynamic Object for the VM.
|
|
impl<T: ScalarValue> Object for ScalarSeries<T> {
|
|
fn type_name(&self) -> &'static str {
|
|
self.type_name
|
|
}
|
|
fn as_any(&self) -> &dyn Any {
|
|
self
|
|
}
|
|
fn as_series(&self) -> Option<&dyn Series> {
|
|
Some(self)
|
|
}
|
|
}
|
|
|
|
impl<T: ScalarValue> Series for ScalarSeries<T> {
|
|
fn get_item(&self, index: usize) -> Option<Value> {
|
|
self.data.borrow().get(index).map(|&v| v.to_value())
|
|
}
|
|
fn len(&self) -> usize {
|
|
self.data.borrow().len()
|
|
}
|
|
fn total_count(&self) -> u64 {
|
|
self.data.borrow().total_count()
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// 2b. Fallback Series for non-scalar types
|
|
// ============================================================================
|
|
|
|
/// A generic series for all non-scalar types (Strings, Tuples, etc.) or mixed data.
|
|
#[derive(Clone)]
|
|
pub struct ValueSeries {
|
|
pub data: std::cell::RefCell<RingBuffer<Value>>,
|
|
}
|
|
|
|
impl ValueSeries {
|
|
pub fn new(lookback_limit: usize) -> Self {
|
|
Self {
|
|
data: std::cell::RefCell::new(RingBuffer::new(lookback_limit)),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Debug for ValueSeries {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
write!(f, "ValueSeries[len: {}]", self.data.borrow().len())
|
|
}
|
|
}
|
|
|
|
impl Object for ValueSeries {
|
|
fn type_name(&self) -> &'static str {
|
|
"ValueSeries"
|
|
}
|
|
fn as_any(&self) -> &dyn Any {
|
|
self
|
|
}
|
|
fn as_series(&self) -> Option<&dyn Series> {
|
|
Some(self)
|
|
}
|
|
}
|
|
|
|
impl Series for ValueSeries {
|
|
fn get_item(&self, index: usize) -> Option<Value> {
|
|
self.data.borrow().get(index).cloned()
|
|
}
|
|
fn len(&self) -> usize {
|
|
self.data.borrow().len()
|
|
}
|
|
fn total_count(&self) -> u64 {
|
|
self.data.borrow().total_count()
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// 3. RecordSeries (SoA - Struct of Arrays)
|
|
// ============================================================================
|
|
|
|
/// An interface allowing iteration over fields of a RecordSeries independently
|
|
/// of the exact type (Type Erasure for member arrays).
|
|
pub trait SeriesMember: Object + Series {
|
|
/// Pushes a dynamic Value into the series, performing the necessary downcast.
|
|
fn push_value(&self, value: Value);
|
|
}
|
|
|
|
impl SeriesMember for ScalarSeries<f64> {
|
|
fn push_value(&self, value: Value) {
|
|
if let Some(v) = value.as_float() {
|
|
self.data.borrow_mut().push(v);
|
|
}
|
|
}
|
|
}
|
|
|
|
impl SeriesMember for ScalarSeries<i64> {
|
|
fn push_value(&self, value: Value) {
|
|
if let Some(v) = value.as_int() {
|
|
self.data.borrow_mut().push(v);
|
|
}
|
|
}
|
|
}
|
|
|
|
impl SeriesMember for ScalarSeries<bool> {
|
|
fn push_value(&self, value: Value) {
|
|
if let Value::Bool(v) = value {
|
|
self.data.borrow_mut().push(v);
|
|
}
|
|
}
|
|
}
|
|
|
|
impl SeriesMember for ValueSeries {
|
|
fn push_value(&self, value: Value) {
|
|
self.data.borrow_mut().push(value);
|
|
}
|
|
}
|
|
|
|
/// The "Struct of Arrays" implementation for Records (e.g., Ticks).
|
|
/// Instead of holding a large array of Records (AoS), this series
|
|
/// splits the record and stores each field in a separate (parallel) series.
|
|
#[derive(Clone)]
|
|
pub struct RecordSeries {
|
|
layout: Arc<RecordLayout>,
|
|
/// Each field of the record gets its own series (SoA).
|
|
/// We use Rc<RefCell> so we can iterate fields individually,
|
|
/// but also pass them as 0-Copy references to indicators!
|
|
fields: Vec<Rc<std::cell::RefCell<dyn SeriesMember>>>,
|
|
}
|
|
|
|
impl RecordSeries {
|
|
/// Creates a new RecordSeries matching the given layout.
|
|
pub fn new(layout: Arc<RecordLayout>, lookback_limit: usize) -> Self {
|
|
let mut fields: Vec<Rc<std::cell::RefCell<dyn SeriesMember>>> =
|
|
Vec::with_capacity(layout.fields.len());
|
|
|
|
for (_, static_type) in &layout.fields {
|
|
// Automatically select the optimal storage representation based on the field's static type.
|
|
let member: Rc<std::cell::RefCell<dyn SeriesMember>> = match static_type {
|
|
StaticType::Float => Rc::new(std::cell::RefCell::new(ScalarSeries::<f64>::new(
|
|
"FloatSeries",
|
|
lookback_limit,
|
|
))),
|
|
StaticType::Int | StaticType::DateTime => Rc::new(std::cell::RefCell::new(
|
|
ScalarSeries::<i64>::new("IntSeries", lookback_limit),
|
|
)),
|
|
StaticType::Bool => Rc::new(std::cell::RefCell::new(ScalarSeries::<bool>::new(
|
|
"BoolSeries",
|
|
lookback_limit,
|
|
))),
|
|
// Fallback for everything else (Text, Lists, nested Records without SoA, etc.)
|
|
_ => Rc::new(std::cell::RefCell::new(ValueSeries::new(lookback_limit))),
|
|
};
|
|
fields.push(member);
|
|
}
|
|
|
|
Self { layout, fields }
|
|
}
|
|
|
|
/// The magical 0-Copy Field Mapper!
|
|
pub fn field(&self, key: Keyword) -> Option<Rc<std::cell::RefCell<dyn SeriesMember>>> {
|
|
self.layout
|
|
.index_of(key)
|
|
.map(|idx| self.fields[idx].clone())
|
|
}
|
|
|
|
/// Dynamically reconstructs the full Record at the given lookback index.
|
|
pub fn get_record(&self, index: usize) -> Option<Value> {
|
|
if self.fields.is_empty() {
|
|
return None;
|
|
}
|
|
|
|
let mut vals = Vec::with_capacity(self.fields.len());
|
|
for field in &self.fields {
|
|
match field.borrow().get_item(index) {
|
|
Some(v) => vals.push(v),
|
|
None => return None, // Out of bounds or incomplete
|
|
}
|
|
}
|
|
|
|
Some(Value::Record(self.layout.clone(), Rc::new(vals)))
|
|
}
|
|
}
|
|
|
|
impl Debug for RecordSeries {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
write!(
|
|
f,
|
|
"RecordSeries[fields: {}, len: {}]",
|
|
self.fields.len(),
|
|
Series::len(self)
|
|
)
|
|
}
|
|
}
|
|
|
|
impl Object for RecordSeries {
|
|
fn type_name(&self) -> &'static str {
|
|
"RecordSeries"
|
|
}
|
|
fn as_any(&self) -> &dyn Any {
|
|
self
|
|
}
|
|
fn as_series(&self) -> Option<&dyn Series> {
|
|
Some(self)
|
|
}
|
|
}
|
|
|
|
impl Series for RecordSeries {
|
|
fn get_item(&self, index: usize) -> Option<Value> {
|
|
self.get_record(index)
|
|
}
|
|
fn len(&self) -> usize {
|
|
self.fields.first().map(|f| f.borrow().len()).unwrap_or(0)
|
|
}
|
|
fn total_count(&self) -> u64 {
|
|
self.fields
|
|
.first()
|
|
.map(|f| f.borrow().total_count())
|
|
.unwrap_or(0)
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// 4. Series View (Safe, 0-Copy access bridge for the VM)
|
|
// ============================================================================
|
|
|
|
#[derive(Clone)]
|
|
pub struct SeriesView {
|
|
pub inner: Rc<std::cell::RefCell<dyn SeriesMember>>,
|
|
pub field_name: Keyword,
|
|
}
|
|
|
|
impl SeriesView {
|
|
pub fn new(inner: Rc<std::cell::RefCell<dyn SeriesMember>>, field_name: Keyword) -> Self {
|
|
Self { inner, field_name }
|
|
}
|
|
}
|
|
|
|
impl Debug for SeriesView {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
write!(
|
|
f,
|
|
"SeriesView[field: {}, len: {}]",
|
|
self.field_name.name(),
|
|
Series::len(self)
|
|
)
|
|
}
|
|
}
|
|
|
|
impl Object for SeriesView {
|
|
fn type_name(&self) -> &'static str {
|
|
"SeriesView"
|
|
}
|
|
fn as_any(&self) -> &dyn Any {
|
|
self
|
|
}
|
|
fn as_series(&self) -> Option<&dyn Series> {
|
|
Some(self)
|
|
}
|
|
}
|
|
|
|
impl Series for SeriesView {
|
|
fn get_item(&self, index: usize) -> Option<Value> {
|
|
self.inner.borrow().get_item(index)
|
|
}
|
|
fn len(&self) -> usize {
|
|
self.inner.borrow().len()
|
|
}
|
|
fn total_count(&self) -> u64 {
|
|
self.inner.borrow().total_count()
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// 4b. Shared Series (Read-Only Reactive Storage)
|
|
// ============================================================================
|
|
|
|
#[derive(Clone)]
|
|
pub struct SharedSeries<T: ScalarValue> {
|
|
pub buffer: Rc<std::cell::RefCell<RingBuffer<T>>>,
|
|
pub type_name: &'static str,
|
|
pub converter: fn(T) -> Value,
|
|
}
|
|
|
|
impl<T: ScalarValue> Debug for SharedSeries<T> {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
write!(
|
|
f,
|
|
"SharedSeries<{}>[len: {}]",
|
|
self.type_name,
|
|
self.buffer.borrow().len()
|
|
)
|
|
}
|
|
}
|
|
|
|
impl<T: ScalarValue> Object for SharedSeries<T> {
|
|
fn type_name(&self) -> &'static str {
|
|
self.type_name
|
|
}
|
|
fn as_any(&self) -> &dyn Any {
|
|
self
|
|
}
|
|
fn as_series(&self) -> Option<&dyn Series> {
|
|
Some(self)
|
|
}
|
|
}
|
|
|
|
impl<T: ScalarValue> Series for SharedSeries<T> {
|
|
fn get_item(&self, index: usize) -> Option<Value> {
|
|
self.buffer
|
|
.borrow()
|
|
.get(index)
|
|
.map(|&v| (self.converter)(v))
|
|
}
|
|
fn len(&self) -> usize {
|
|
self.buffer.borrow().len()
|
|
}
|
|
fn total_count(&self) -> u64 {
|
|
self.buffer.borrow().total_count()
|
|
}
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub struct SharedValueSeries {
|
|
pub buffer: Rc<std::cell::RefCell<RingBuffer<Value>>>,
|
|
}
|
|
|
|
impl Debug for SharedValueSeries {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
write!(f, "SharedValueSeries[len: {}]", self.buffer.borrow().len())
|
|
}
|
|
}
|
|
|
|
impl Object for SharedValueSeries {
|
|
fn type_name(&self) -> &'static str {
|
|
"SharedValueSeries"
|
|
}
|
|
fn as_any(&self) -> &dyn Any {
|
|
self
|
|
}
|
|
fn as_series(&self) -> Option<&dyn Series> {
|
|
Some(self)
|
|
}
|
|
}
|
|
|
|
impl Series for SharedValueSeries {
|
|
fn get_item(&self, index: usize) -> Option<Value> {
|
|
self.buffer.borrow().get(index).cloned()
|
|
}
|
|
fn len(&self) -> usize {
|
|
self.buffer.borrow().len()
|
|
}
|
|
fn total_count(&self) -> u64 {
|
|
self.buffer.borrow().total_count()
|
|
}
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub struct SharedRecordSeries {
|
|
pub layout: Arc<RecordLayout>,
|
|
pub field_buffers: Vec<Rc<std::cell::RefCell<dyn SeriesMember>>>,
|
|
}
|
|
|
|
impl Debug for SharedRecordSeries {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
write!(
|
|
f,
|
|
"SharedRecordSeries[fields: {}, len: {}]",
|
|
self.field_buffers.len(),
|
|
Series::len(self)
|
|
)
|
|
}
|
|
}
|
|
|
|
impl Object for SharedRecordSeries {
|
|
fn type_name(&self) -> &'static str {
|
|
"SharedRecordSeries"
|
|
}
|
|
fn as_any(&self) -> &dyn Any {
|
|
self
|
|
}
|
|
fn as_series(&self) -> Option<&dyn Series> {
|
|
Some(self)
|
|
}
|
|
}
|
|
|
|
impl Series for SharedRecordSeries {
|
|
fn get_item(&self, index: usize) -> Option<Value> {
|
|
if self.field_buffers.is_empty() {
|
|
return None;
|
|
}
|
|
|
|
let mut vals = Vec::with_capacity(self.field_buffers.len());
|
|
for buffer in &self.field_buffers {
|
|
match buffer.borrow().get_item(index) {
|
|
Some(v) => vals.push(v),
|
|
None => return None,
|
|
}
|
|
}
|
|
|
|
Some(Value::Record(self.layout.clone(), Rc::new(vals)))
|
|
}
|
|
fn len(&self) -> usize {
|
|
self.field_buffers
|
|
.first()
|
|
.map(|f| f.borrow().len())
|
|
.unwrap_or(0)
|
|
}
|
|
fn total_count(&self) -> u64 {
|
|
self.field_buffers
|
|
.first()
|
|
.map(|f| f.borrow().total_count())
|
|
.unwrap_or(0)
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// 5. Script Integration (RTL Registration)
|
|
// ============================================================================
|
|
|
|
pub fn register(env: &Environment) {
|
|
// (series lookback_limit template_or_type_keyword) -> Series
|
|
env.register_native_fn(
|
|
"series",
|
|
StaticType::Function(Box::new(Signature {
|
|
params: StaticType::Tuple(vec![StaticType::Int, StaticType::Any]),
|
|
ret: StaticType::Any,
|
|
})),
|
|
Purity::Impure,
|
|
|args: &[Value]| {
|
|
let lookback_limit = args[0].as_int().unwrap_or(0) as usize;
|
|
let arg = &args[1];
|
|
match arg {
|
|
Value::Keyword(k) => {
|
|
match k.name().to_lowercase().as_str() {
|
|
"float" => Value::Object(Rc::new(ScalarSeries::<f64>::new("FloatSeries", lookback_limit))),
|
|
"int" | "datetime" => Value::Object(Rc::new(ScalarSeries::<i64>::new("IntSeries", lookback_limit))),
|
|
"bool" => Value::Object(Rc::new(ScalarSeries::<bool>::new("BoolSeries", lookback_limit))),
|
|
"text" => Value::Object(Rc::new(ValueSeries::new(lookback_limit))),
|
|
_ => panic!("Unknown or unsupported series type keyword: :{}", k.name()),
|
|
}
|
|
}
|
|
Value::Record(schema_layout, schema_values) => {
|
|
// Interpret the record as a schema: { :field :type_keyword }
|
|
let mut fields = Vec::with_capacity(schema_layout.fields.len());
|
|
for (i, (name, _)) in schema_layout.fields.iter().enumerate() {
|
|
let type_keyword = match &schema_values[i] {
|
|
Value::Keyword(tk) => tk.name().to_lowercase(),
|
|
_ => panic!("Field :{} in series schema must be a type keyword (e.g., :float, :int)", name.name()),
|
|
};
|
|
let ty = match type_keyword.as_str() {
|
|
"float" => StaticType::Float,
|
|
"int" => StaticType::Int,
|
|
"bool" => StaticType::Bool,
|
|
"datetime" => StaticType::DateTime,
|
|
"text" => StaticType::Text,
|
|
_ => panic!("Unsupported type keyword :{} for series field :{}", type_keyword, name.name()),
|
|
};
|
|
fields.push((*name, ty));
|
|
}
|
|
let final_layout = RecordLayout::get_or_create(fields);
|
|
Value::Object(Rc::new(RecordSeries::new(final_layout, lookback_limit)))
|
|
}
|
|
_ => panic!("series expects a lookback_limit and a type keyword (:float, :int...) or a schema record (e.g., {{:price :float}})"),
|
|
}
|
|
},
|
|
);
|
|
|
|
// (push series value) -> Void
|
|
env.register_native_fn(
|
|
"push",
|
|
StaticType::Function(Box::new(Signature {
|
|
params: StaticType::Tuple(vec![StaticType::Any, StaticType::Any]),
|
|
ret: StaticType::Void,
|
|
})),
|
|
Purity::Impure,
|
|
|args: &[Value]| {
|
|
let (series_val, val) = (&args[0], &args[1]);
|
|
|
|
if let Value::Object(obj) = series_val {
|
|
let any = obj.as_any();
|
|
|
|
// Polymorphic push logic
|
|
if let Some(rs) = any.downcast_ref::<RecordSeries>() {
|
|
if let Value::Record(_, values) = val {
|
|
for (i, v) in values.iter().enumerate() {
|
|
if let Some(f) = rs.fields.get(i) {
|
|
f.borrow().push_value(v.clone());
|
|
}
|
|
}
|
|
} else {
|
|
panic!("push to RecordSeries expects a record");
|
|
}
|
|
} else if let Some(s) = any.downcast_ref::<ScalarSeries<f64>>() {
|
|
s.push_value(val.clone());
|
|
} else if let Some(s) = any.downcast_ref::<ScalarSeries<i64>>() {
|
|
s.push_value(val.clone());
|
|
} else if let Some(s) = any.downcast_ref::<ScalarSeries<bool>>() {
|
|
s.push_value(val.clone());
|
|
} else if let Some(s) = any.downcast_ref::<ValueSeries>() {
|
|
s.push_value(val.clone());
|
|
} else {
|
|
panic!("Object is not a mutable series");
|
|
}
|
|
return Value::Void;
|
|
}
|
|
panic!("push expects a series object as the first argument")
|
|
},
|
|
);
|
|
|
|
// (len series) -> Int
|
|
env.register_native_fn(
|
|
"len",
|
|
StaticType::Function(Box::new(Signature {
|
|
params: StaticType::Tuple(vec![StaticType::Any]),
|
|
ret: StaticType::Int,
|
|
})),
|
|
Purity::Pure,
|
|
|args: &[Value]| {
|
|
if let Value::Object(obj) = &args[0]
|
|
&& let Some(s) = obj.as_series()
|
|
{
|
|
return Value::Int(s.len() as i64);
|
|
}
|
|
Value::Int(0)
|
|
},
|
|
);
|
|
}
|