Brummel/RustAst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Specialize pipeline types for performance

Browse Source 
Introduces type specialization for reactive pipeline nodes and their
data buffers. This eliminates the overhead of generic `Value` enums and
`SharedValueSeries` when dealing with known scalar or record types.

The architecture shifts buffer instantiation from the VM to the runtime
(RTL), leveraging type information from the AST. A new `out_type` field
is added to `BoundKind::Pipe` to store the static type of the pipeline's
output, determined by the type checker.

This enables the creation of specialized `RingBuffer<T>` and
`SharedRecordSeries` (for Struct-of-Arrays layout) when the output type
is known, significantly improving memory usage and processing speed for
time-series data, especially in financial analysis.
This commit is contained in:
Michael Schimmel
2026-03-02 22:03:24 +01:00
parent a4af142719
commit f7cb6655af
14 changed files with 295 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ast/vm.rs
+24 -44
View File
@@ -368,10 +368,13 @@ impl VM {
Ok(Value::Void)
}
}
BoundKind::Pipe { inputs, lambda } => {
use crate::ast::rtl::streams::{PipelineNode, PipeStream, StreamNode, ValuePusher, ObservableStream};
use crate::ast::rtl::series::{RingBuffer, SharedValueSeries};
BoundKind::Pipe {
inputs,
lambda,
out_type,
} => {
use crate::ast::rtl::streams::{PipelineNode, StreamNode};
let mut obs_streams = Vec::new();
for input in inputs {
let val = self.eval_internal(obs, input)?;
@@ -381,7 +384,10 @@ impl VM {
} else if let Some(p) = obj.as_any().downcast_ref::<PipelineNode>() {
obs_streams.push(p.stream.clone());
} else {
return Err(format!("Pipe input must be a stream, found {}", obj.type_name()));
return Err(format!(
"Pipe input must be a stream, found {}",
obj.type_name()
));
}
} else {
return Err("Pipe input must be an object (stream)".to_string());
@@ -398,46 +404,20 @@ impl VM {
// Create the persistent execution closure for the PipeStream
let mut pipe_vm = crate::ast::vm::VM::new(self.globals.clone());
let my_closure = lambda_obj.clone();
let executor: Box<dyn FnMut(Vec<Value>) -> Value> = Box::new(move |args: Vec<Value>| -> Value {
match pipe_vm.run_with_args(my_closure.clone(), args) {
Ok(res) => res,
Err(e) => panic!("Pipeline lambda execution failed: {}", e),
}
});
let executor: Box<dyn FnMut(Vec<Value>) -> Value> =
Box::new(move |args: Vec<Value>| -> Value {
match pipe_vm.run_with_args(my_closure.clone(), args) {
Ok(res) => res,
Err(e) => panic!("Pipeline lambda execution failed: {}", e),
}
});
let pipe = Rc::new(RefCell::new(PipeStream::new(
"pipe".to_string(),
inputs.len(),
Some(executor)
)));
for (i, stream) in obs_streams.into_iter().enumerate() {
let adapter = Rc::new(RefCell::new(crate::ast::rtl::streams::SourceAdapter {
target: pipe.clone() as Rc<RefCell<dyn crate::ast::rtl::streams::Observer>>,
target_index: i,
}));
stream.add_observer(adapter);
}
// Create the data buffer and pusher for the pipe's output
let buffer = Rc::new(RefCell::new(RingBuffer::<Value>::new()));
let pusher = Rc::new(RefCell::new(ValuePusher {
buffer: buffer.clone(),
lookback: Some(100), // Default lookback for now
}));
// The pipe pushes to the buffer
pipe.borrow().add_observer(pusher);
// Create the Series view for script access
let series = Rc::new(SharedValueSeries {
buffer,
});
let node = Rc::new(PipelineNode {
stream: pipe,
series,
});
// Delegate to the RTL Factory for specialized buffer instantiation
let node = crate::ast::rtl::streams::build_pipeline_node(
obs_streams,
executor,
out_type,
);
Ok(Value::Object(node))
}