Adding Pipes

ASTPlayground/ASTPlayground.dpr

@@ -39,7 +39,7 @@ uses
     Myc.Fmx.AstEditor.Handlers.Data in '..\Src\AST\Myc.Fmx.AstEditor.Handlers.Data.pas',
     Myc.Ast.RTL.TypeRegistry in '..\Src\AST\Myc.Ast.RTL.TypeRegistry.pas',
     Myc.Trade.Broker in '..\Src\Myc.Trade.Broker.pas',
-    Myc.Ast.Pipes in '..\Src\AST\Myc.Ast.Pipes.pas';
+    Myc.Data.Stream.Pipes in '..\Src\Data\Myc.Data.Stream.Pipes.pas';
 
 {$R *.res}
 

ASTPlayground/ASTPlayground.dproj

@@ -169,7 +169,7 @@
         <DCCReference Include="..\Src\AST\Myc.Fmx.AstEditor.Handlers.Data.pas"/>
         <DCCReference Include="..\Src\AST\Myc.Ast.RTL.TypeRegistry.pas"/>
         <DCCReference Include="..\Src\Myc.Trade.Broker.pas"/>
-        <DCCReference Include="..\Src\AST\Myc.Ast.Pipes.pas"/>
+        <DCCReference Include="..\Src\Data\Myc.Data.Stream.Pipes.pas"/>
         <BuildConfiguration Include="Base">
             <Key>Base</Key>
         </BuildConfiguration>

Src/Data/Myc.Data.Scalar.pas

@@ -199,13 +199,14 @@ type
         function GetRecordCount: Int64;
         function GetTotalCount: Int64;
         {$endregion}
-        procedure Add(const Item: IKeywordMapping<TScalar>; Lookback: Int64 = -1);
+        procedure Add(const Item: IKeywordMapping<TScalar>; Lookback: Int64 = -1); overload;
+        procedure Add(const Items: TArray<TScalar.TValue>; Lookback: Int64 = -1); overload;
         property RecordCount: Int64 read GetRecordCount;
         property TotalCount: Int64 read GetTotalCount;
     end;
 
     // A series of scalar records, optimized for memory and access speed.
-    TScalarRecordSeries = class(TInterfacedObject, IWriteableScalarRecordSeries, IKeywordMapping<ISeries>)
+    TScalarRecordSeries = class(TInterfacedObject, IKeywordMapping<ISeries>, IScalarRecordSeries, IWriteableScalarRecordSeries)
         type
             TMemberSeries = class(TGenericContainedObject<TScalarRecordSeries>, ISeries)
             private
@@ -239,7 +240,8 @@ type
         constructor Create(const ADef: IScalarRecordDefinition);
         destructor Destroy; override;
 
-        procedure Add(const Item: IKeywordMapping<TScalar>; Lookback: Int64 = -1);
+        procedure Add(const Item: IKeywordMapping<TScalar>; Lookback: Int64 = -1); overload;
+        procedure Add(const Items: TArray<TScalar.TValue>; Lookback: Int64 = -1); overload;
 
         property RecordCount: Int64 read GetRecordCount;
         property Def: IScalarRecordDefinition read GetDef;
@@ -816,9 +818,24 @@ end;
 
 procedure TScalarRecordSeries.Add(const Item: IKeywordMapping<TScalar>; Lookback: Int64 = -1);
 begin
+    Assert(Item.Count = FDef.Count, 'Mapping does not fit series definition');
+
     var lb := FDef.Count * Integer(Lookback);
     for var i := 0 to FDef.Count - 1 do
+    begin
+        Assert(Item[i].Key = FDef[i].Key, 'Mapping does not fit series definition');
         FArray.Add(Item[i].Value.Value, lb);
+    end;
+    inc(FTotalCount);
+end;
+
+procedure TScalarRecordSeries.Add(const Items: TArray<TScalar.TValue>; Lookback: Int64 = -1);
+begin
+    Assert(Length(Items) = FDef.Count, 'Array does not fit series definition');
+
+    var lb := FDef.Count * Integer(Lookback);
+    for var i := 0 to FDef.Count - 1 do
+        FArray.Add(Items[i], lb);
     inc(FTotalCount);
 end;
 

Src/Data/Myc.Data.Stream.Pipes.pas

@@ -33,7 +33,7 @@ Ich m
 )
 *)
 
-unit Myc.Ast.Pipes;
+unit Myc.Data.Stream.Pipes;
 
 interface
 
@@ -42,7 +42,6 @@ uses
     System.Classes,
     System.Generics.Collections,
     System.SyncObjs,
-    Myc.Data.Value,
     Myc.Data.Scalar,
     Myc.Data.Keyword,
     Myc.Data.Stream,
@@ -77,6 +76,8 @@ type
     // The executable instance of a pipe.
     // Acts as IStream (Producer) for downstream consumers and manages internal inputs.
     TPipeStream = class(TInterfacedObject, IStream)
+        type
+            TProc = reference to procedure(const Series: TArray<ISeries>);
     private
         FConfig: TPipeConfig;
         FSources: TArray<TPipeSource>;
@@ -88,18 +89,16 @@ type
 
         // Execution Logic
         // The compiled lambda function representing (fn [inputs...] ...)
-        FLambda: TDataValue.TFunc;
+        FLambda: TProc;
-        FLambdaArgs: TArray<TDataValue>;
+        FLambdaArgs: TArray<ISeries>;
 
         procedure CheckBarrierAndFire(CurrentCycle: Int64);
         function GetSeries: IScalarRecordSeries;
 
     public
-        constructor Create(const AConfig: TPipeConfig; const ASources: TArray<IStream>);
+        constructor Create(const AConfig: TPipeConfig; const ASources: TArray<IStream>; const ALambda: TProc);
         destructor Destroy; override;
 
-        procedure SetTransformation(const Lambda: TDataValue.TFunc);
-
         // Injected into lambda scope as "emit"
         procedure Emit(const Value: IScalarRecord);
 
@@ -110,9 +109,6 @@ type
 
 implementation
 
-uses
-    Myc.Ast.Visitor;
-
 { TPipeSource }
 
 constructor TPipeSource.Create(AOwner: TPipeStream; ASource: IStream; const AInputs: TArray<TScalarRecordField>);
@@ -142,12 +138,13 @@ end;
 
 { TPipeStream }
 
-constructor TPipeStream.Create(const AConfig: TPipeConfig; const ASources: TArray<IStream>);
+constructor TPipeStream.Create(const AConfig: TPipeConfig; const ASources: TArray<IStream>; const ALambda: TProc);
 var
     fields: TArray<TScalarRecordField>;
 begin
     inherited Create;
     FConfig := AConfig;
+    FLambda := ALambda;
     FLastFiredCycleID := -1;
 
     Assert(Length(AConfig) = Length(ASources));
@@ -172,7 +169,7 @@ begin
         begin
             fields[n] := AConfig[i][j];
 
-            FLambdaArgs[n] := TDataValue.FromSeries(ASources[i].Series[j].Value);
+            FLambdaArgs[n] := ASources[i].Series[j].Value;
 
             inc(n);
         end;
@@ -192,11 +189,6 @@ begin
     inherited;
 end;
 
-procedure TPipeStream.SetTransformation(const Lambda: TDataValue.TFunc);
-begin
-    FLambda := Lambda;
-end;
-
 function TPipeStream.Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
 begin
     FObservers.Lock;

Src/Data/Myc.Data.Stream.pas

@@ -8,6 +8,7 @@ uses
     System.Generics.Collections,
     System.SyncObjs,
     Myc.Data.Scalar,
+    Myc.Data.Keyword,
     Myc.Data.Value,
     Myc.Core.Notifier;
 
@@ -47,8 +48,7 @@ type
         {$region 'private'}
         function GetStream: IStream;
         {$endregion}
-
+        procedure Push(const Value: IKeywordMapping<TScalar>);
-        procedure Push(const Value: IScalarRecord);
         property Stream: IStream read GetStream;
     end;
 
@@ -73,11 +73,11 @@ type
             private
                 FName: string;
                 FSeries: IWriteableScalarRecordSeries;
-                FQueue: TQueue<IScalarRecord>;
+                FQueue: TQueue<IKeywordMapping<TScalar>>;
                 FQueueLock: TSpinLock;
                 FObservers: TStreamNotifier;
 
-                procedure Push(const Value: IScalarRecord);
+                procedure Push(const Value: IKeywordMapping<TScalar>);
                 function GetStream: IStream;
                 function Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
                 procedure Unsubscribe(Tag: TSubscriptionTag);
@@ -125,7 +125,7 @@ begin
     inherited Create;
     FName := AName;
     FSeries := ASeries;
-    FQueue := TQueue<IScalarRecord>.Create;
+    FQueue := TQueue<IKeywordMapping<TScalar>>.Create;
 end;
 
 destructor TGraphExecutor.TChannel.Destroy;
@@ -135,10 +135,8 @@ begin
     inherited;
 end;
 
-procedure TGraphExecutor.TChannel.Push(const Value: IScalarRecord);
+procedure TGraphExecutor.TChannel.Push(const Value: IKeywordMapping<TScalar>);
 begin
-    Assert(Value.Def = FSeries.Def);
-
     FQueueLock.Enter;
     try
         FQueue.Enqueue(Value);

Src/Myc.Data.Records.pas

@@ -18,7 +18,7 @@ type
                 FFieldType: TFieldType;
                 FTypeInfo: PtypeInfo;
                 FName: string;
-                d FOffset: Integer;
+                FOffset: Integer;
                 FSize: Integer;
 
                 procedure FromType(const [ref] Buffer: TBytes; const Src);

Src/Myc.Trade.DataFeed.pas

@@ -168,6 +168,7 @@ type
         function GetCount: Integer;
         function GetItems(Idx: Integer): TPair<IKeyword, TScalar>;
         function GetFields(const Key: IKeyword): TScalar;
+        function GetDef: IScalarRecordDefinition;
     public
         constructor Create(const ADef: IScalarRecordDefinition);
         procedure SetData(const AData: TScalar.PValue); inline;
@@ -191,6 +192,11 @@ begin
     Result := FDef.Count;
 end;
 
+function TScalarRecordView.GetDef: IScalarRecordDefinition;
+begin
+    Result := FDef;
+end;
+
 function TScalarRecordView.GetFields(const Key: IKeyword): TScalar;
 var
     idx: Integer;