From 85d21943dd9f0525e6ec40212edb8924deeaf8d7 Mon Sep 17 00:00:00 2001 From: Michael Schimmel Date: Sat, 28 Feb 2026 18:30:44 +0100 Subject: [PATCH] Add series module for time-series data Introduce the `series` module to handle time-series data storage and manipulation. This module includes: - `RingBuffer`: An efficient ring buffer for storing time-series data with a configurable lookback. - `ScalarSeries`: A type-safe series for primitive scalar values (f64, i64, bool) optimized for performance. - `ValueSeries`: A fallback series for non-scalar or mixed data types. - `RecordSeries`: A "Struct of Arrays" implementation for records, splitting fields into parallel series for efficient access. --- docs/todo/record_optimizations_plan.md | 37 ---- src/ast/rtl/mod.rs | 3 +- src/ast/rtl/series.rs | 225 +++++++++++++++++++++++++ 3 files changed, 227 insertions(+), 38 deletions(-) delete mode 100644 docs/todo/record_optimizations_plan.md create mode 100644 src/ast/rtl/series.rs diff --git a/docs/todo/record_optimizations_plan.md b/docs/todo/record_optimizations_plan.md deleted file mode 100644 index e50f2db..0000000 --- a/docs/todo/record_optimizations_plan.md +++ /dev/null @@ -1,37 +0,0 @@ -# Record Optimization Plan - -Dieses Dokument beschreibt geplante Optimierungen für Records im Myc-Compiler, um die Performance bei Datenstrukturen zu steigern. - -## 1. Constant Folding für Record-Literale - -**Status:** Aktuell werden Record-Literale in `engine.rs` nur rekursiv besucht, aber nicht gefaltet. - -**Ziel:** Wenn alle Felder (Keys und Values) eines Records Konstanten sind, soll der gesamte Record in eine `BoundKind::Constant(Value::Record(...))` transformiert werden. - -**Umsetzung:** -- In `src/ast/compiler/optimizer/folder.rs` eine Methode `try_fold_record` implementieren. -- In `src/ast/compiler/optimizer/engine.rs` im Case `BoundKind::Record` diese Methode aufrufen. -- Dies ermöglicht es dem bereits existierenden Folder für `BoundKind::GetField`, Feldzugriffe auf Literalen direkt zur Kompilierzeit aufzulösen. - -## 2. Record Inlining in SubstitutionMap - -**Status:** Records werden derzeit nicht aggressiv in die `SubstitutionMap` aufgenommen, wenn sie Variablen zugewiesen werden. - -**Ziel:** Zuweisungen von (konstanten) Records an Variablen sollen in `sub.values` gespeichert werden. - -**Beispiel:** -```lisp -(def r {:x 10 :y 20}) -(.x r) ; Sollte zu 10 optimiert werden -``` - -**Umsetzung:** -- Sicherstellen, dass `Value::Record` von `Inliner::is_inlinable_value` als sicher eingestuft wird. -- Testen, ob `BoundKind::GetField` bei einem `Get` auf eine Variable, die in `sub.values` als Record bekannt ist, den Wert extrahieren kann. - -## 3. Propagation von Record-Layouts - -**Ziel:** Wenn das Layout eines Records statisch bekannt ist (auch wenn die Werte nicht konstant sind), könnten Feldzugriffe (`GetField`) effizienter vorbereitet werden, um die Laufzeit-Suche im Layout-Hash/Index zu minimieren. - ---- -*Erstellt am 26. Februar 2026 zur Nachverfolgung der Optimizer-Verbesserungen.* diff --git a/src/ast/rtl/mod.rs b/src/ast/rtl/mod.rs index ec23604..91b6d64 100644 --- a/src/ast/rtl/mod.rs +++ b/src/ast/rtl/mod.rs @@ -2,6 +2,7 @@ pub mod core; pub mod datetime; pub mod intrinsics; pub mod math; +pub mod series; pub mod type_registry; use crate::ast::environment::Environment; @@ -10,4 +11,4 @@ pub fn register(env: &Environment) { core::register(env); datetime::register(env); math::register(env); -} +} \ No newline at end of file diff --git a/src/ast/rtl/series.rs b/src/ast/rtl/series.rs new file mode 100644 index 0000000..0841415 --- /dev/null +++ b/src/ast/rtl/series.rs @@ -0,0 +1,225 @@ +use std::any::Any; +use std::collections::VecDeque; +use std::fmt::{self, Debug}; +use std::rc::Rc; +use std::sync::Arc; + +use crate::ast::types::{Keyword, Object, RecordLayout, StaticType, Value}; + +// ============================================================================ +// 1. Core Storage Engine +// ============================================================================ + +/// A generic, efficient ring buffer for time series data. +/// (Replaces Delphi's TChunkArray for now with VecDeque for O(1) push/pop) +#[derive(Clone)] +pub struct RingBuffer { + buffer: VecDeque, + total_count: u64, +} + +impl Default for RingBuffer { + fn default() -> Self { + Self::new() + } +} + +impl RingBuffer { + pub fn new() -> Self { + Self { + buffer: VecDeque::new(), + total_count: 0, + } + } + + /// Adds an element. Removes the oldest if the lookback limit is reached. + #[inline] + pub fn push(&mut self, item: T, lookback: Option) { + self.buffer.push_back(item); + self.total_count += 1; + + if let Some(limit) = lookback { + while self.buffer.len() > limit { + self.buffer.pop_front(); + } + } + } + + /// Access in "Financial Style": Index 0 is the newest element. + #[inline] + pub fn get(&self, index: usize) -> Option<&T> { + let len = self.buffer.len(); + if index < len { + self.buffer.get(len - 1 - index) + } else { + None + } + } + + #[inline] + pub fn total_count(&self) -> u64 { + self.total_count + } + + #[inline] + pub fn len(&self) -> usize { + self.buffer.len() + } + + pub fn is_empty(&self) -> bool { + self.buffer.is_empty() + } +} + +// ============================================================================ +// 2. Trait & Primitives for Fast-Paths +// ============================================================================ + +/// Marker trait ensuring our fast arrays only store true, flat scalars +/// (No pointers/heaps like String or Record). +pub trait ScalarValue: Copy + Debug + 'static {} + +impl ScalarValue for f64 {} +impl ScalarValue for i64 {} +impl ScalarValue for bool {} + +/// A highly optimized, type-safe series for primitive values (Float, Int, Bool). +#[derive(Clone)] +pub struct ScalarSeries { + pub data: RingBuffer, + pub type_name: &'static str, +} + +impl ScalarSeries { + pub fn new(type_name: &'static str) -> Self { + Self { + data: RingBuffer::new(), + type_name, + } + } +} + +impl Debug for ScalarSeries { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "ScalarSeries<{}>[len: {}]", self.type_name, self.data.len()) + } +} + +// Makes the series usable as a dynamic Object for the VM. +impl Object for ScalarSeries { + fn type_name(&self) -> &'static str { + self.type_name + } + fn as_any(&self) -> &dyn Any { + self + } +} + +// ============================================================================ +// 2b. Fallback Series for non-scalar types +// ============================================================================ + +/// A generic series for all non-scalar types (Strings, Tuples, etc.) or mixed data. +#[derive(Clone)] +pub struct ValueSeries { + pub data: RingBuffer, +} + +impl Default for ValueSeries { + fn default() -> Self { + Self::new() + } +} + +impl ValueSeries { + pub fn new() -> Self { + Self { + data: RingBuffer::new(), + } + } +} + +impl Debug for ValueSeries { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "ValueSeries[len: {}]", self.data.len()) + } +} + +impl Object for ValueSeries { + fn type_name(&self) -> &'static str { + "ValueSeries" + } + fn as_any(&self) -> &dyn Any { + self + } +} + +// ============================================================================ +// 3. RecordSeries (SoA - Struct of Arrays) +// ============================================================================ + +/// An interface allowing iteration over fields of a RecordSeries independently +/// of the exact type (Type Erasure for member arrays). +pub trait SeriesMember: Object { + // We could add type-safe `push_any(&mut self, Value)` here later, + // if the fallback path of the VM needs to insert values dynamically. +} + +impl SeriesMember for ScalarSeries {} +impl SeriesMember for ValueSeries {} + +/// The "Struct of Arrays" implementation for Records (e.g., Ticks). +/// Instead of holding a large array of Records (AoS), this series +/// splits the record and stores each field in a separate (parallel) series. +#[derive(Clone)] +pub struct RecordSeries { + layout: Arc, + /// Each field of the record gets its own series (SoA). + /// We use Rc so we can iterate fields individually, + /// but also pass them as 0-Copy references to indicators! + fields: Vec>>, +} + +impl RecordSeries { + /// Creates a new RecordSeries matching the given layout. + pub fn new(layout: Arc) -> Self { + let mut fields: Vec>> = Vec::with_capacity(layout.fields.len()); + + for (_, static_type) in &layout.fields { + // Automatically select the optimal storage representation based on the field's static type. + let member: Rc> = match static_type { + StaticType::Float => Rc::new(std::cell::RefCell::new(ScalarSeries::::new("FloatSeries"))), + StaticType::Int | StaticType::DateTime => Rc::new(std::cell::RefCell::new(ScalarSeries::::new("IntSeries"))), + StaticType::Bool => Rc::new(std::cell::RefCell::new(ScalarSeries::::new("BoolSeries"))), + // Fallback for everything else (Text, Lists, nested Records without SoA, etc.) + _ => Rc::new(std::cell::RefCell::new(ValueSeries::new())), + }; + fields.push(member); + } + + Self { layout, fields } + } + + /// The magical 0-Copy Field Mapper! + /// Returns the complete series of a single field. + /// Example: `my_ticks.field(Keyword::intern("close"))` + /// returns an `Rc>>`. No copy involved! + pub fn field(&self, key: Keyword) -> Option>> { + self.layout.index_of(key).map(|idx| self.fields[idx].clone()) + } +} + +impl Debug for RecordSeries { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "RecordSeries[fields: {}]", self.fields.len()) + } +} + +impl Object for RecordSeries { + fn type_name(&self) -> &'static str { + "RecordSeries" + } + fn as_any(&self) -> &dyn Any { + self + } +}