Add series module for time-series data
Introduce the `series` module to handle time-series data storage and manipulation. This module includes: - `RingBuffer`: An efficient ring buffer for storing time-series data with a configurable lookback. - `ScalarSeries`: A type-safe series for primitive scalar values (f64, i64, bool) optimized for performance. - `ValueSeries`: A fallback series for non-scalar or mixed data types. - `RecordSeries`: A "Struct of Arrays" implementation for records, splitting fields into parallel series for efficient access.
This commit is contained in:
@@ -1,37 +0,0 @@
|
|||||||
# Record Optimization Plan
|
|
||||||
|
|
||||||
Dieses Dokument beschreibt geplante Optimierungen für Records im Myc-Compiler, um die Performance bei Datenstrukturen zu steigern.
|
|
||||||
|
|
||||||
## 1. Constant Folding für Record-Literale
|
|
||||||
|
|
||||||
**Status:** Aktuell werden Record-Literale in `engine.rs` nur rekursiv besucht, aber nicht gefaltet.
|
|
||||||
|
|
||||||
**Ziel:** Wenn alle Felder (Keys und Values) eines Records Konstanten sind, soll der gesamte Record in eine `BoundKind::Constant(Value::Record(...))` transformiert werden.
|
|
||||||
|
|
||||||
**Umsetzung:**
|
|
||||||
- In `src/ast/compiler/optimizer/folder.rs` eine Methode `try_fold_record` implementieren.
|
|
||||||
- In `src/ast/compiler/optimizer/engine.rs` im Case `BoundKind::Record` diese Methode aufrufen.
|
|
||||||
- Dies ermöglicht es dem bereits existierenden Folder für `BoundKind::GetField`, Feldzugriffe auf Literalen direkt zur Kompilierzeit aufzulösen.
|
|
||||||
|
|
||||||
## 2. Record Inlining in SubstitutionMap
|
|
||||||
|
|
||||||
**Status:** Records werden derzeit nicht aggressiv in die `SubstitutionMap` aufgenommen, wenn sie Variablen zugewiesen werden.
|
|
||||||
|
|
||||||
**Ziel:** Zuweisungen von (konstanten) Records an Variablen sollen in `sub.values` gespeichert werden.
|
|
||||||
|
|
||||||
**Beispiel:**
|
|
||||||
```lisp
|
|
||||||
(def r {:x 10 :y 20})
|
|
||||||
(.x r) ; Sollte zu 10 optimiert werden
|
|
||||||
```
|
|
||||||
|
|
||||||
**Umsetzung:**
|
|
||||||
- Sicherstellen, dass `Value::Record` von `Inliner::is_inlinable_value` als sicher eingestuft wird.
|
|
||||||
- Testen, ob `BoundKind::GetField` bei einem `Get` auf eine Variable, die in `sub.values` als Record bekannt ist, den Wert extrahieren kann.
|
|
||||||
|
|
||||||
## 3. Propagation von Record-Layouts
|
|
||||||
|
|
||||||
**Ziel:** Wenn das Layout eines Records statisch bekannt ist (auch wenn die Werte nicht konstant sind), könnten Feldzugriffe (`GetField`) effizienter vorbereitet werden, um die Laufzeit-Suche im Layout-Hash/Index zu minimieren.
|
|
||||||
|
|
||||||
---
|
|
||||||
*Erstellt am 26. Februar 2026 zur Nachverfolgung der Optimizer-Verbesserungen.*
|
|
||||||
+2
-1
@@ -2,6 +2,7 @@ pub mod core;
|
|||||||
pub mod datetime;
|
pub mod datetime;
|
||||||
pub mod intrinsics;
|
pub mod intrinsics;
|
||||||
pub mod math;
|
pub mod math;
|
||||||
|
pub mod series;
|
||||||
pub mod type_registry;
|
pub mod type_registry;
|
||||||
|
|
||||||
use crate::ast::environment::Environment;
|
use crate::ast::environment::Environment;
|
||||||
@@ -10,4 +11,4 @@ pub fn register(env: &Environment) {
|
|||||||
core::register(env);
|
core::register(env);
|
||||||
datetime::register(env);
|
datetime::register(env);
|
||||||
math::register(env);
|
math::register(env);
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,225 @@
|
|||||||
|
use std::any::Any;
|
||||||
|
use std::collections::VecDeque;
|
||||||
|
use std::fmt::{self, Debug};
|
||||||
|
use std::rc::Rc;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use crate::ast::types::{Keyword, Object, RecordLayout, StaticType, Value};
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// 1. Core Storage Engine
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/// A generic, efficient ring buffer for time series data.
|
||||||
|
/// (Replaces Delphi's TChunkArray for now with VecDeque for O(1) push/pop)
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct RingBuffer<T> {
|
||||||
|
buffer: VecDeque<T>,
|
||||||
|
total_count: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> Default for RingBuffer<T> {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> RingBuffer<T> {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
buffer: VecDeque::new(),
|
||||||
|
total_count: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Adds an element. Removes the oldest if the lookback limit is reached.
|
||||||
|
#[inline]
|
||||||
|
pub fn push(&mut self, item: T, lookback: Option<usize>) {
|
||||||
|
self.buffer.push_back(item);
|
||||||
|
self.total_count += 1;
|
||||||
|
|
||||||
|
if let Some(limit) = lookback {
|
||||||
|
while self.buffer.len() > limit {
|
||||||
|
self.buffer.pop_front();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Access in "Financial Style": Index 0 is the newest element.
|
||||||
|
#[inline]
|
||||||
|
pub fn get(&self, index: usize) -> Option<&T> {
|
||||||
|
let len = self.buffer.len();
|
||||||
|
if index < len {
|
||||||
|
self.buffer.get(len - 1 - index)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub fn total_count(&self) -> u64 {
|
||||||
|
self.total_count
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub fn len(&self) -> usize {
|
||||||
|
self.buffer.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_empty(&self) -> bool {
|
||||||
|
self.buffer.is_empty()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// 2. Trait & Primitives for Fast-Paths
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/// Marker trait ensuring our fast arrays only store true, flat scalars
|
||||||
|
/// (No pointers/heaps like String or Record).
|
||||||
|
pub trait ScalarValue: Copy + Debug + 'static {}
|
||||||
|
|
||||||
|
impl ScalarValue for f64 {}
|
||||||
|
impl ScalarValue for i64 {}
|
||||||
|
impl ScalarValue for bool {}
|
||||||
|
|
||||||
|
/// A highly optimized, type-safe series for primitive values (Float, Int, Bool).
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct ScalarSeries<T: ScalarValue> {
|
||||||
|
pub data: RingBuffer<T>,
|
||||||
|
pub type_name: &'static str,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: ScalarValue> ScalarSeries<T> {
|
||||||
|
pub fn new(type_name: &'static str) -> Self {
|
||||||
|
Self {
|
||||||
|
data: RingBuffer::new(),
|
||||||
|
type_name,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: ScalarValue> Debug for ScalarSeries<T> {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
write!(f, "ScalarSeries<{}>[len: {}]", self.type_name, self.data.len())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Makes the series usable as a dynamic Object for the VM.
|
||||||
|
impl<T: ScalarValue> Object for ScalarSeries<T> {
|
||||||
|
fn type_name(&self) -> &'static str {
|
||||||
|
self.type_name
|
||||||
|
}
|
||||||
|
fn as_any(&self) -> &dyn Any {
|
||||||
|
self
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// 2b. Fallback Series for non-scalar types
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/// A generic series for all non-scalar types (Strings, Tuples, etc.) or mixed data.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct ValueSeries {
|
||||||
|
pub data: RingBuffer<Value>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for ValueSeries {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ValueSeries {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
data: RingBuffer::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Debug for ValueSeries {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
write!(f, "ValueSeries[len: {}]", self.data.len())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Object for ValueSeries {
|
||||||
|
fn type_name(&self) -> &'static str {
|
||||||
|
"ValueSeries"
|
||||||
|
}
|
||||||
|
fn as_any(&self) -> &dyn Any {
|
||||||
|
self
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// 3. RecordSeries (SoA - Struct of Arrays)
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/// An interface allowing iteration over fields of a RecordSeries independently
|
||||||
|
/// of the exact type (Type Erasure for member arrays).
|
||||||
|
pub trait SeriesMember: Object {
|
||||||
|
// We could add type-safe `push_any(&mut self, Value)` here later,
|
||||||
|
// if the fallback path of the VM needs to insert values dynamically.
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: ScalarValue> SeriesMember for ScalarSeries<T> {}
|
||||||
|
impl SeriesMember for ValueSeries {}
|
||||||
|
|
||||||
|
/// The "Struct of Arrays" implementation for Records (e.g., Ticks).
|
||||||
|
/// Instead of holding a large array of Records (AoS), this series
|
||||||
|
/// splits the record and stores each field in a separate (parallel) series.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct RecordSeries {
|
||||||
|
layout: Arc<RecordLayout>,
|
||||||
|
/// Each field of the record gets its own series (SoA).
|
||||||
|
/// We use Rc<RefCell> so we can iterate fields individually,
|
||||||
|
/// but also pass them as 0-Copy references to indicators!
|
||||||
|
fields: Vec<Rc<std::cell::RefCell<dyn SeriesMember>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RecordSeries {
|
||||||
|
/// Creates a new RecordSeries matching the given layout.
|
||||||
|
pub fn new(layout: Arc<RecordLayout>) -> Self {
|
||||||
|
let mut fields: Vec<Rc<std::cell::RefCell<dyn SeriesMember>>> = Vec::with_capacity(layout.fields.len());
|
||||||
|
|
||||||
|
for (_, static_type) in &layout.fields {
|
||||||
|
// Automatically select the optimal storage representation based on the field's static type.
|
||||||
|
let member: Rc<std::cell::RefCell<dyn SeriesMember>> = match static_type {
|
||||||
|
StaticType::Float => Rc::new(std::cell::RefCell::new(ScalarSeries::<f64>::new("FloatSeries"))),
|
||||||
|
StaticType::Int | StaticType::DateTime => Rc::new(std::cell::RefCell::new(ScalarSeries::<i64>::new("IntSeries"))),
|
||||||
|
StaticType::Bool => Rc::new(std::cell::RefCell::new(ScalarSeries::<bool>::new("BoolSeries"))),
|
||||||
|
// Fallback for everything else (Text, Lists, nested Records without SoA, etc.)
|
||||||
|
_ => Rc::new(std::cell::RefCell::new(ValueSeries::new())),
|
||||||
|
};
|
||||||
|
fields.push(member);
|
||||||
|
}
|
||||||
|
|
||||||
|
Self { layout, fields }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The magical 0-Copy Field Mapper!
|
||||||
|
/// Returns the complete series of a single field.
|
||||||
|
/// Example: `my_ticks.field(Keyword::intern("close"))`
|
||||||
|
/// returns an `Rc<RefCell<ScalarSeries<f64>>>`. No copy involved!
|
||||||
|
pub fn field(&self, key: Keyword) -> Option<Rc<std::cell::RefCell<dyn SeriesMember>>> {
|
||||||
|
self.layout.index_of(key).map(|idx| self.fields[idx].clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Debug for RecordSeries {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
write!(f, "RecordSeries[fields: {}]", self.fields.len())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Object for RecordSeries {
|
||||||
|
fn type_name(&self) -> &'static str {
|
||||||
|
"RecordSeries"
|
||||||
|
}
|
||||||
|
fn as_any(&self) -> &dyn Any {
|
||||||
|
self
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user