feat: Specialize pipeline types for performance
Introduces type specialization for reactive pipeline nodes and their data buffers. This eliminates the overhead of generic `Value` enums and `SharedValueSeries` when dealing with known scalar or record types. The architecture shifts buffer instantiation from the VM to the runtime (RTL), leveraging type information from the AST. A new `out_type` field is added to `BoundKind::Pipe` to store the static type of the pipeline's output, determined by the type checker. This enables the creation of specialized `RingBuffer<T>` and `SharedRecordSeries` (for Struct-of-Arrays layout) when the output type is known, significantly improving memory usage and processing speed for time-series data, especially in financial analysis.
This commit is contained in:
@@ -237,7 +237,7 @@ impl<'a> Analyzer<'a> {
|
||||
Purity::Impure,
|
||||
)
|
||||
}
|
||||
BoundKind::Pipe { inputs, lambda } => {
|
||||
BoundKind::Pipe { inputs, lambda, out_type } => {
|
||||
let mut analyzed_inputs = Vec::with_capacity(inputs.len());
|
||||
for input in inputs {
|
||||
analyzed_inputs.push(self.visit(Rc::new(input.clone())));
|
||||
@@ -247,6 +247,7 @@ impl<'a> Analyzer<'a> {
|
||||
BoundKind::Pipe {
|
||||
inputs: analyzed_inputs,
|
||||
lambda: a_lambda,
|
||||
out_type: out_type.clone(),
|
||||
},
|
||||
Purity::Impure,
|
||||
)
|
||||
|
||||
@@ -287,6 +287,7 @@ impl Binder {
|
||||
BoundKind::Pipe {
|
||||
inputs: bound_inputs,
|
||||
lambda: bound_lambda,
|
||||
out_type: crate::ast::types::StaticType::Any,
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
@@ -138,6 +138,7 @@ pub enum BoundKind<T = ()> {
|
||||
Pipe {
|
||||
inputs: Vec<BoundNode<T>>,
|
||||
lambda: Box<BoundNode<T>>,
|
||||
out_type: crate::ast::types::StaticType,
|
||||
},
|
||||
Block {
|
||||
exprs: Vec<BoundNode<T>>,
|
||||
|
||||
@@ -89,7 +89,7 @@ impl CapturePass {
|
||||
args: Box::new(Self::transform(*args, capture_map)),
|
||||
};
|
||||
}
|
||||
BoundKind::Pipe { inputs, lambda } => {
|
||||
BoundKind::Pipe { inputs, lambda, out_type } => {
|
||||
let mut t_inputs = Vec::with_capacity(inputs.len());
|
||||
for input in inputs {
|
||||
t_inputs.push(Self::transform(input, capture_map));
|
||||
@@ -97,6 +97,7 @@ impl CapturePass {
|
||||
node.kind = BoundKind::Pipe {
|
||||
inputs: t_inputs,
|
||||
lambda: Box::new(Self::transform(*lambda, capture_map)),
|
||||
out_type: out_type.clone(),
|
||||
};
|
||||
}
|
||||
BoundKind::Block { exprs } => {
|
||||
|
||||
@@ -203,7 +203,7 @@ impl Dumper {
|
||||
self.visit(args);
|
||||
self.indent -= 1;
|
||||
}
|
||||
BoundKind::Pipe { inputs, lambda } => {
|
||||
BoundKind::Pipe { inputs, lambda, .. } => {
|
||||
self.log("Pipe", node);
|
||||
self.indent += 1;
|
||||
for input in inputs {
|
||||
|
||||
@@ -386,7 +386,7 @@ impl Optimizer {
|
||||
)
|
||||
}
|
||||
|
||||
BoundKind::Pipe { inputs, lambda } => {
|
||||
BoundKind::Pipe { inputs, lambda, out_type } => {
|
||||
let mut o_inputs = Vec::with_capacity(inputs.len());
|
||||
for input in inputs {
|
||||
o_inputs.push(self.visit_node(input, sub, path));
|
||||
@@ -396,6 +396,7 @@ impl Optimizer {
|
||||
BoundKind::Pipe {
|
||||
inputs: o_inputs,
|
||||
lambda: o_lambda,
|
||||
out_type: out_type.clone(),
|
||||
},
|
||||
node.ty.clone(),
|
||||
)
|
||||
|
||||
@@ -152,7 +152,7 @@ impl UsageInfo {
|
||||
BoundKind::Again { args } => {
|
||||
self.collect(args);
|
||||
}
|
||||
BoundKind::Pipe { inputs, lambda } => {
|
||||
BoundKind::Pipe { inputs, lambda, .. } => {
|
||||
for input in inputs {
|
||||
self.collect(input);
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ impl TCO {
|
||||
args: Box::new(Self::transform(Rc::new((**args).clone()), false)),
|
||||
}
|
||||
}
|
||||
BoundKind::Pipe { inputs, lambda } => {
|
||||
BoundKind::Pipe { inputs, lambda, out_type } => {
|
||||
let mut t_inputs = Vec::with_capacity(inputs.len());
|
||||
for input in inputs {
|
||||
t_inputs.push(Self::transform(Rc::new(input.clone()), false));
|
||||
@@ -56,6 +56,7 @@ impl TCO {
|
||||
BoundKind::Pipe {
|
||||
inputs: t_inputs,
|
||||
lambda: Box::new(Self::transform(Rc::new((**lambda).clone()), false)),
|
||||
out_type: out_type.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -387,12 +387,24 @@ impl TypeChecker {
|
||||
)
|
||||
}
|
||||
|
||||
BoundKind::Pipe { inputs, lambda } => {
|
||||
BoundKind::Pipe { inputs, lambda, .. } => {
|
||||
let mut typed_inputs = Vec::with_capacity(inputs.len());
|
||||
let mut arg_types = Vec::with_capacity(inputs.len());
|
||||
for input in inputs {
|
||||
typed_inputs.push(self.check_node(input, ctx)?);
|
||||
let typed_input = self.check_node(input, ctx)?;
|
||||
// Unwrap Series(T) to T for the lambda argument
|
||||
let arg_ty = if let StaticType::Series(inner) = &typed_input.ty {
|
||||
*inner.clone()
|
||||
} else {
|
||||
StaticType::Any
|
||||
};
|
||||
arg_types.push(arg_ty);
|
||||
typed_inputs.push(typed_input);
|
||||
}
|
||||
let typed_lambda = self.check_node(*lambda, ctx)?;
|
||||
|
||||
// Specialized check for the lambda using the input types!
|
||||
let typed_lambda = self.check(*lambda, &arg_types)?;
|
||||
|
||||
let ret_ty = if let StaticType::Function(sig) = &typed_lambda.ty {
|
||||
// If the lambda returns an Optional(T), the pipeline filters Void and stores T!
|
||||
if let StaticType::Optional(inner) = &sig.ret {
|
||||
@@ -407,6 +419,7 @@ impl TypeChecker {
|
||||
BoundKind::Pipe {
|
||||
inputs: typed_inputs,
|
||||
lambda: Box::new(typed_lambda),
|
||||
out_type: ret_ty.clone(),
|
||||
},
|
||||
StaticType::Series(Box::new(ret_ty)),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user