Streams refactoring

ASTPlayground/ASTPlayground.dpr

@@ -38,7 +38,6 @@ uses
     Myc.Fmx.AstEditor.Handlers.Data in '..\Src\AST\Myc.Fmx.AstEditor.Handlers.Data.pas',
     Myc.Ast.RTL.TypeRegistry in '..\Src\AST\Myc.Ast.RTL.TypeRegistry.pas',
     Myc.Trade.Broker in '..\Src\Myc.Trade.Broker.pas',
-    Myc.Data.Stream.Pipes in '..\Src\Data\Myc.Data.Stream.Pipes.pas',
     Myc.Data.Stream in '..\Src\Data\Myc.Data.Stream.pas',
     Myc.Fmx.AstEditor.Handlers.Pipes in '..\Src\AST\Myc.Fmx.AstEditor.Handlers.Pipes.pas',
     Demo.Finance in '..\Test\Demo.Finance.pas',

ASTPlayground/ASTPlayground.dproj

@@ -168,7 +168,6 @@
         <DCCReference Include="..\Src\AST\Myc.Fmx.AstEditor.Handlers.Data.pas"/>
         <DCCReference Include="..\Src\AST\Myc.Ast.RTL.TypeRegistry.pas"/>
         <DCCReference Include="..\Src\Myc.Trade.Broker.pas"/>
-        <DCCReference Include="..\Src\Data\Myc.Data.Stream.Pipes.pas"/>
         <DCCReference Include="..\Src\Data\Myc.Data.Stream.pas"/>
         <DCCReference Include="..\Src\AST\Myc.Fmx.AstEditor.Handlers.Pipes.pas"/>
         <DCCReference Include="..\Test\Demo.Finance.pas"/>

Src/AST/Myc.Ast.Evaluator.pas

@@ -62,7 +62,6 @@ uses
     Myc.Data.Decimal,
     Myc.Data.Series,
     Myc.Data.Stream,
-    Myc.Data.Stream.Pipes,
     Myc.Ast.Types;
 
 type
@@ -602,9 +601,7 @@ begin
     // Compile the transformation function
     lambdaFunc := Visit(N.Transformation).AsMethod();
 
-    // WARNING: If this is executing dynamically (without TypeChecker), StaticType is Unknown.
+    // Allow both stRecord and stRecordSeries (as TypeChecker correctly assigns stRecordSeries for Pipe nodes)
-    // Pipe creation REQUIRES the output definition.
-    // Fixed: Allow both stRecord and stRecordSeries (as TypeChecker correctly assigns stRecordSeries for Pipe nodes)
     if (N.StaticType.Kind <> stRecordSeries) and (N.StaticType.Kind <> stRecord) then
         raise EEvaluatorException.Create('Pipe requires Type Checking to determine output structure (RecordDefinition).');
 

Src/Data/Myc.Data.Stream.Pipes.pas

@@ -1,280 +0,0 @@
-unit Myc.Data.Stream.Pipes;
-
-interface
-
-uses
-    System.SysUtils,
-    System.Classes,
-    System.Generics.Collections,
-    System.SyncObjs,
-    Myc.Data.Scalar,
-    Myc.Data.Keyword,
-    Myc.Data.Stream,
-    Myc.Core.Notifier;
-
-type
-    // =========================================================================
-    // BASE STREAM
-    // Implements storage (Series) and broadcasting (Observers).
-    // Does not define WHEN data is emitted (Policy).
-    // =========================================================================
-    TCustomDataStream = class(TInterfacedObject, IStream)
-    strict private
-        FSeries: IWriteableScalarRecordSeries;
-        FObservers: TMycNotifyList<IStreamObserver>;
-        function GetSeries: IScalarRecordSeries;
-    protected
-        // Derived classes call this to write data and notify listeners.
-        // ACycleID: The cycle this data belongs to.
-        procedure Emit(const Value: array of TScalar.TValue; ACycleID: Int64);
-    public
-        constructor Create(const ADef: IScalarRecordDefinition);
-        destructor Destroy; override;
-
-        // IStream
-        function Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
-        procedure Unsubscribe(Tag: TSubscriptionTag);
-        property Series: IScalarRecordSeries read GetSeries;
-    end;
-
-    // =========================================================================
-    // ROOT STREAM (SOURCE)
-    // Acts as the clock source. Generates new CycleIDs.
-    // =========================================================================
-    TRootStream = class(TCustomDataStream)
-    private
-        FLastCycleID: Int64;
-    public
-        constructor Create(const ADef: IScalarRecordDefinition);
-        // Public API to inject data from Delphi
-        procedure Push(const RowData: TArray<TScalar.TValue>);
-    end;
-
-    // =========================================================================
-    // PIPE STREAM (NODE)
-    // Reacts to upstream signals using barrier synchronization.
-    // =========================================================================
-    TPipeStream = class; // Forward
-
-    TPipeConfig = TArray<TArray<TScalarRecordField>>;
-
-    TPipeSource = class(TContainedObject, IStreamObserver)
-    private
-        FSource: IStream;
-        FTag: TSubscriptionTag;
-        FLastSeenCycle: Int64;
-        procedure OnSignal(const Signal: TStreamSignal);
-    public
-        constructor Create(AOwner: TPipeStream; ASource: IStream);
-        destructor Destroy; override;
-        property LastSeenCycle: Int64 read FLastSeenCycle;
-    end;
-
-    TPipeStream = class(TCustomDataStream)
-    public
-        type
-            TPipeLambda = reference to function(const Sources: array of ISeries; out Results: array of TScalar.TValue): Boolean;
-    private
-        FSources: TArray<TPipeSource>;
-        FSourceSeries: TArray<ISeries>;
-        FLastFiredCycleID: Int64;
-        FLambda: TPipeLambda;
-
-        procedure CheckBarrierAndFire(CurrentCycle: Int64);
-    public
-        constructor Create(
-            const AConfig: TPipeConfig;
-            const ADef: IScalarRecordDefinition;
-            const ASources: TArray<IStream>;
-            const ALambda: TPipeLambda
-        );
-        destructor Destroy; override;
-    end;
-
-implementation
-
-{ TCustomDataStream }
-
-constructor TCustomDataStream.Create(const ADef: IScalarRecordDefinition);
-begin
-    inherited Create;
-    FSeries := TScalarRecordSeries.Create(ADef);
-end;
-
-destructor TCustomDataStream.Destroy;
-begin
-    FObservers.Finalize;
-    inherited;
-end;
-
-function TCustomDataStream.GetSeries: IScalarRecordSeries;
-begin
-    Result := FSeries;
-end;
-
-function TCustomDataStream.Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
-begin
-    FObservers.Lock;
-    try
-        Result := FObservers.Advise(Observer);
-    finally
-        FObservers.Release;
-    end;
-end;
-
-procedure TCustomDataStream.Unsubscribe(Tag: TSubscriptionTag);
-begin
-    FObservers.Lock;
-    try
-        FObservers.Unadvise(Tag);
-    finally
-        FObservers.Release;
-    end;
-end;
-
-procedure TCustomDataStream.Emit(const Value: array of TScalar.TValue; ACycleID: Int64);
-var
-    signal: TStreamSignal;
-begin
-    FObservers.Lock;
-    try
-        // 1. Write Data
-        FSeries.Add(Value);
-
-        // 2. Broadcast Signal
-        signal := TStreamSignal.Create(skData, ACycleID);
-
-        FObservers.Notify(
-            function(const Obs: IStreamObserver): Boolean
-            begin
-                Obs.OnSignal(signal);
-                Result := True;
-            end
-        );
-    finally
-        FObservers.Release;
-    end;
-end;
-
-{ TRootStream }
-
-constructor TRootStream.Create(const ADef: IScalarRecordDefinition);
-begin
-    inherited Create(ADef);
-    FLastCycleID := 0;
-end;
-
-procedure TRootStream.Push(const RowData: TArray<TScalar.TValue>);
-begin
-    // Root streams increment the global clock
-    Inc(FLastCycleID);
-    Emit(RowData, FLastCycleID);
-end;
-
-{ TPipeSource }
-
-constructor TPipeSource.Create(AOwner: TPipeStream; ASource: IStream);
-begin
-    inherited Create(AOwner);
-    FSource := ASource;
-    FLastSeenCycle := -1;
-    FTag := FSource.Subscribe(Self);
-end;
-
-destructor TPipeSource.Destroy;
-begin
-    FSource.Unsubscribe(FTag);
-    inherited;
-end;
-
-procedure TPipeSource.OnSignal(const Signal: TStreamSignal);
-begin
-    if Signal.Kind = skData then
-    begin
-        FLastSeenCycle := Signal.CycleID;
-        (Controller as TPipeStream).CheckBarrierAndFire(FLastSeenCycle);
-    end;
-end;
-
-{ TPipeStream }
-
-constructor TPipeStream.Create(
-    const AConfig: TPipeConfig;
-    const ADef: IScalarRecordDefinition;
-    const ASources: TArray<IStream>;
-    const ALambda: TPipeLambda
-);
-var
-    i, j, n: Integer;
-begin
-    // Pass Definition to base class
-    inherited Create(ADef);
-
-    FLambda := ALambda;
-    FLastFiredCycleID := -1;
-
-    SetLength(FSources, Length(ASources));
-
-    // Flatten Sources
-    n := 0;
-    for i := 0 to High(AConfig) do
-        inc(n, Length(AConfig[i]));
-    SetLength(FSourceSeries, n);
-
-    n := 0;
-    for i := 0 to High(ASources) do
-    begin
-        FSources[i] := TPipeSource.Create(Self, ASources[i]);
-        for j := 0 to High(AConfig[i]) do
-        begin
-            // Extract the specific column series
-            FSourceSeries[n] := ASources[i].Series.Fields[AConfig[i][j].Key];
-            inc(n);
-        end;
-    end;
-end;
-
-destructor TPipeStream.Destroy;
-begin
-    for var i := High(FSources) downto 0 do
-        FSources[i].Free;
-    FSources := nil;
-    FSourceSeries := nil;
-    inherited;
-end;
-
-procedure TPipeStream.CheckBarrierAndFire(CurrentCycle: Int64);
-var
-    resultVal: TArray<TScalar.TValue>;
-begin
-    // We reuse the FObservers lock from the base class to protect state
-    // But since FObservers is private, we access it via method or need to make it protected.
-    // Making it protected or using a separate lock is cleaner.
-    // For now, let's assume we rely on the fact that OnSignal is usually serialized per thread
-    // or add a lock.
-    // -> Ideally, TCustomDataStream should expose Lock/Unlock or we add a Lock here.
-
-    // Simplification: We assume thread safety is handled by the caller or we add a dedicated lock.
-    // For this example, let's just do the logic:
-
-    for var src in FSources do
-        if src.LastSeenCycle < CurrentCycle then
-            exit; // Barrier closed
-
-    if FLastFiredCycleID >= CurrentCycle then
-        exit; // Already fired
-
-    FLastFiredCycleID := CurrentCycle;
-
-    if Assigned(FLambda) then
-    begin
-        SetLength(resultVal, Series.Def.Count);
-        if FLambda(FSourceSeries, resultVal) then
-        begin
-            // Pass through CycleID from upstream
-            Emit(resultVal, FLastFiredCycleID);
-        end;
-    end;
-end;
-
-end.

Src/Data/Myc.Data.Stream.pas

@@ -43,61 +43,82 @@ type
         property Series: IScalarRecordSeries read GetSeries;
     end;
 
-    IInputChannel = interface
+    // =========================================================================
-        {$region 'private'}
+    // BASE STREAM
-        function GetStream: IStream;
+    // Implements storage (Series) and broadcasting (Observers).
-        {$endregion}
+    // Does not define WHEN data is emitted (Policy).
-        procedure Push(const Value: IKeywordMapping<TScalar>);
+    // =========================================================================
-        property Stream: IStream read GetStream;
+    TCustomDataStream = class(TInterfacedObject, IStream)
-    end;
+    strict private
-
+        FSeries: IWriteableScalarRecordSeries;
-    IGraphExecutor = interface
+        FObservers: TMycNotifyList<IStreamObserver>;
-        {$region 'private'}
+        function GetSeries: IScalarRecordSeries;
-        function GetCycleID: Int64;
+    protected
-        {$endregion}
+        // Derived classes call this to write data and notify listeners.
-
+        // ACycleID: The cycle this data belongs to.
-        function GetInputChannel(const Name: string; const Def: IScalarRecordDefinition): IInputChannel;
+        procedure Emit(const Value: array of TScalar.TValue; ACycleID: Int64);
-        procedure Step;
-        property CycleID: Int64 read GetCycleID;
-    end;
-
-    TGraphExecutor = class(TInterfacedObject, IGraphExecutor)
-    private
-        type
-            TChannel = class(TInterfacedObject, IInputChannel, IStream)
-            private
-                type
-                    TStreamNotifier = TMycNotifyList<IStreamObserver>;
-            private
-                FName: string;
-                FSeries: IWriteableScalarRecordSeries;
-                FQueue: TQueue<IKeywordMapping<TScalar>>;
-                FQueueLock: TSpinLock;
-                FObservers: TStreamNotifier;
-
-                procedure Push(const Value: IKeywordMapping<TScalar>);
-                function GetStream: IStream;
-                function Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
-                procedure Unsubscribe(Tag: TSubscriptionTag);
-                procedure Emit(ACycle: Int64);
-
-                function GetSeries: IScalarRecordSeries;
-
-            public
-                constructor Create(const AName: string; const ASeries: IWriteableScalarRecordSeries);
-                destructor Destroy; override;
-            end;
-
-    private
-        FCycleID: Int64;
-        FChannels: TDictionary<string, IInputChannel>;
-        FLock: TSpinLock;
-        function GetInputChannel(const Name: string; const Def: IScalarRecordDefinition): IInputChannel;
-        function GetCycleID: Int64;
     public
-        constructor Create;
+        constructor Create(const ADef: IScalarRecordDefinition);
+        destructor Destroy; override;
+
+        // IStream
+        function Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
+        procedure Unsubscribe(Tag: TSubscriptionTag);
+        property Series: IScalarRecordSeries read GetSeries;
+    end;
+
+    // =========================================================================
+    // ROOT STREAM (SOURCE)
+    // Acts as the clock source. Generates new CycleIDs.
+    // =========================================================================
+    TRootStream = class(TCustomDataStream)
+    private
+        FLastCycleID: Int64;
+    public
+        constructor Create(const ADef: IScalarRecordDefinition);
+        // Public API to inject data from Delphi
+        procedure Push(const RowData: TArray<TScalar.TValue>);
+    end;
+
+    // =========================================================================
+    // PIPE STREAM (NODE)
+    // Reacts to upstream signals using barrier synchronization.
+    // =========================================================================
+    TPipeStream = class; // Forward
+
+    TPipeConfig = TArray<TArray<TScalarRecordField>>;
+
+    TPipeSource = class(TContainedObject, IStreamObserver)
+    private
+        FSource: IStream;
+        FTag: TSubscriptionTag;
+        FLastSeenCycle: Int64;
+        procedure OnSignal(const Signal: TStreamSignal);
+    public
+        constructor Create(AOwner: TPipeStream; ASource: IStream);
+        destructor Destroy; override;
+        property LastSeenCycle: Int64 read FLastSeenCycle;
+    end;
+
+    TPipeStream = class(TCustomDataStream)
+    public
+        type
+            TPipeLambda = reference to function(const Sources: array of ISeries; out Results: array of TScalar.TValue): Boolean;
+    private
+        FSources: TArray<TPipeSource>;
+        FSourceSeries: TArray<ISeries>;
+        FLastFiredCycleID: Int64;
+        FLambda: TPipeLambda;
+
+        procedure CheckBarrierAndFire(CurrentCycle: Int64);
+    public
+        constructor Create(
+            const AConfig: TPipeConfig;
+            const ADef: IScalarRecordDefinition;
+            const ASources: TArray<IStream>;
+            const ALambda: TPipeLambda
+        );
         destructor Destroy; override;
-        procedure Step;
     end;
 
 implementation
@@ -119,39 +140,26 @@ begin
         Result := Format('Signal(Heartbeat, #%d)', [CycleID]);
 end;
 
-{ TGraphExecutor.TChannel }
+{ TCustomDataStream }
 
-constructor TGraphExecutor.TChannel.Create(const AName: string; const ASeries: IWriteableScalarRecordSeries);
+constructor TCustomDataStream.Create(const ADef: IScalarRecordDefinition);
 begin
     inherited Create;
-    FName := AName;
+    FSeries := TScalarRecordSeries.Create(ADef);
-    FSeries := ASeries;
-    FQueue := TQueue<IKeywordMapping<TScalar>>.Create;
 end;
 
-destructor TGraphExecutor.TChannel.Destroy;
+destructor TCustomDataStream.Destroy;
 begin
     FObservers.Finalize;
-    FQueue.Free;
     inherited;
 end;
 
-procedure TGraphExecutor.TChannel.Push(const Value: IKeywordMapping<TScalar>);
+function TCustomDataStream.GetSeries: IScalarRecordSeries;
 begin
-    FQueueLock.Enter;
+    Result := FSeries;
-    try
-        FQueue.Enqueue(Value);
-    finally
-        FQueueLock.Exit;
-    end;
 end;
 
-function TGraphExecutor.TChannel.GetStream: IStream;
+function TCustomDataStream.Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
-begin
-    Result := Self;
-end;
-
-function TGraphExecutor.TChannel.Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
 begin
     FObservers.Lock;
     try
@@ -161,7 +169,7 @@ begin
     end;
 end;
 
-procedure TGraphExecutor.TChannel.Unsubscribe(Tag: TSubscriptionTag);
+procedure TCustomDataStream.Unsubscribe(Tag: TSubscriptionTag);
 begin
     FObservers.Lock;
     try
@@ -171,25 +179,18 @@ begin
     end;
 end;
 
-procedure TGraphExecutor.TChannel.Emit(ACycle: Int64);
+procedure TCustomDataStream.Emit(const Value: array of TScalar.TValue; ACycleID: Int64);
 var
     signal: TStreamSignal;
 begin
-    FQueueLock.Enter;
-    try
-        FSeries.Add(FQueue.Dequeue);
-        signal :=
-            TStreamSignal.Create(
-                if FQueue.Count > 0 then skData
-                else skHeartbeat,
-                ACycle
-            )
-    finally
-        FQueueLock.Exit;
-    end;
-
     FObservers.Lock;
     try
+        // 1. Write Data
+        FSeries.Add(Value);
+
+        // 2. Broadcast Signal
+        signal := TStreamSignal.Create(skData, ACycleID);
+
         FObservers.Notify(
             function(const Obs: IStreamObserver): Boolean
             begin
@@ -202,63 +203,125 @@ begin
     end;
 end;
 
-function TGraphExecutor.TChannel.GetSeries: IScalarRecordSeries;
+{ TRootStream }
+
+constructor TRootStream.Create(const ADef: IScalarRecordDefinition);
 begin
-    Result := FSeries;
+    inherited Create(ADef);
+    FLastCycleID := 0;
 end;
 
-{ TGraphExecutor }
+procedure TRootStream.Push(const RowData: TArray<TScalar.TValue>);
-
-constructor TGraphExecutor.Create;
 begin
-    inherited Create;
+    // Root streams increment the global clock
-    FChannels := TDictionary<string, IInputChannel>.Create;
+    Inc(FLastCycleID);
-    FCycleID := 0;
+    Emit(RowData, FLastCycleID);
 end;
 
-destructor TGraphExecutor.Destroy;
+{ TPipeSource }
+
+constructor TPipeSource.Create(AOwner: TPipeStream; ASource: IStream);
 begin
-    FChannels.Free;
+    inherited Create(AOwner);
+    FSource := ASource;
+    FLastSeenCycle := -1;
+    FTag := FSource.Subscribe(Self);
+end;
+
+destructor TPipeSource.Destroy;
+begin
+    FSource.Unsubscribe(FTag);
     inherited;
 end;
 
-function TGraphExecutor.GetInputChannel(const Name: string; const Def: IScalarRecordDefinition): IInputChannel;
+procedure TPipeSource.OnSignal(const Signal: TStreamSignal);
-var
-    impl: TChannel;
 begin
-    FLock.Enter;
+    if Signal.Kind = skData then
-    try
+    begin
-        if not FChannels.TryGetValue(Name, Result) then
+        FLastSeenCycle := Signal.CycleID;
+        (Controller as TPipeStream).CheckBarrierAndFire(FLastSeenCycle);
+    end;
+end;
+
+{ TPipeStream }
+
+constructor TPipeStream.Create(
+    const AConfig: TPipeConfig;
+    const ADef: IScalarRecordDefinition;
+    const ASources: TArray<IStream>;
+    const ALambda: TPipeLambda
+);
+var
+    i, j, n: Integer;
+begin
+    // Pass Definition to base class
+    inherited Create(ADef);
+
+    FLambda := ALambda;
+    FLastFiredCycleID := -1;
+
+    SetLength(FSources, Length(ASources));
+
+    // Flatten Sources
+    n := 0;
+    for i := 0 to High(AConfig) do
+        inc(n, Length(AConfig[i]));
+    SetLength(FSourceSeries, n);
+
+    n := 0;
+    for i := 0 to High(ASources) do
+    begin
+        FSources[i] := TPipeSource.Create(Self, ASources[i]);
+        for j := 0 to High(AConfig[i]) do
         begin
-            impl := TChannel.Create(Name, TScalarRecordSeries.Create(Def));
+            // Extract the specific column series
-            Result := impl;
+            FSourceSeries[n] := ASources[i].Series.Fields[AConfig[i][j].Key];
-            FChannels.Add(Name, Result);
+            inc(n);
         end;
-    finally
-        FLock.Exit;
     end;
 end;
 
-function TGraphExecutor.GetCycleID: Int64;
+destructor TPipeStream.Destroy;
 begin
-    Result := FCycleID;
+    for var i := High(FSources) downto 0 do
+        FSources[i].Free;
+    FSources := nil;
+    FSourceSeries := nil;
+    inherited;
 end;
 
-procedure TGraphExecutor.Step;
+procedure TPipeStream.CheckBarrierAndFire(CurrentCycle: Int64);
 var
-    channelsArr: TArray<IInputChannel>;
+    resultVal: TArray<TScalar.TValue>;
-    channel: IInputChannel;
 begin
-    FLock.Enter;
+    // We reuse the FObservers lock from the base class to protect state
-    try
+    // But since FObservers is private, we access it via method or need to make it protected.
-        Inc(FCycleID);
+    // Making it protected or using a separate lock is cleaner.
-        channelsArr := FChannels.Values.ToArray;
+    // For now, let's assume we rely on the fact that OnSignal is usually serialized per thread
-    finally
+    // or add a lock.
-        FLock.Exit;
+    // -> Ideally, TCustomDataStream should expose Lock/Unlock or we add a Lock here.
-    end;
 
-    for channel in channelsArr do
+    // Simplification: We assume thread safety is handled by the caller or we add a dedicated lock.
-        (channel as TChannel).Emit(FCycleID);
+    // For this example, let's just do the logic:
+
+    for var src in FSources do
+        if src.LastSeenCycle < CurrentCycle then
+            exit; // Barrier closed
+
+    if FLastFiredCycleID >= CurrentCycle then
+        exit; // Already fired
+
+    FLastFiredCycleID := CurrentCycle;
+
+    if Assigned(FLambda) then
+    begin
+        SetLength(resultVal, Series.Def.Count);
+        if FLambda(FSourceSeries, resultVal) then
+        begin
+            // Pass through CycleID from upstream
+            Emit(resultVal, FLastFiredCycleID);
+        end;
+    end;
 end;
 
 initialization

Test/AST/Test.Myc.Ast.Stream.Pipes.pas

@@ -1,313 +0,0 @@
-unit Test.Myc.Ast.Stream.Pipes;
-
-interface
-
-uses
-    DUnitX.TestFramework,
-    System.SysUtils,
-    System.Generics.Collections,
-    // Core Units
-    Myc.Data.Scalar,
-    Myc.Data.Keyword,
-    Myc.Data.Value,
-    Myc.Data.Series,
-    Myc.Data.Stream,
-    Myc.Data.Stream.Pipes,
-    // AST & Compiler
-    Myc.Ast,
-    Myc.Ast.Nodes,
-    Myc.Ast.Scope,
-    Myc.Ast.Types,
-    Myc.Ast.Identities,
-    Myc.Ast.Script,
-    Myc.Ast.Compiler.Binder,
-    Myc.Ast.Compiler.TypeChecker,
-    Myc.Ast.Evaluator;
-
-type
-    // Helper: Captures compiler errors so we can assert on them
-    TCapturingLog = class(TInterfacedObject, ICompilerLog)
-    private
-        FErrors: TList<string>;
-    public
-        constructor Create;
-        destructor Destroy; override;
-        procedure Add(ALevel: TCompilerErrorLevel; const AMessage: string; const ANode: IAstNode = nil);
-        procedure AddError(const AMessage: string; const ANode: IAstNode = nil);
-        procedure AddWarning(const AMessage: string; const ANode: IAstNode = nil);
-        function HasErrors: Boolean;
-        function GetEntryCount: Integer;
-        function GetEntries: TArray<TCompilerError>;
-        property Errors: TList<string> read FErrors;
-    end;
-
-    // Helper: Collects stream output for assertions
-    TDataCollector = class(TInterfacedObject, IStreamObserver)
-    private
-        FStream: IStream;
-        FLastRecord: IScalarRecord;
-        FSignalCount: Integer;
-    public
-        constructor Create(AStream: IStream);
-        procedure OnSignal(const Signal: TStreamSignal);
-        property LastRecord: IScalarRecord read FLastRecord;
-        property SignalCount: Integer read FSignalCount;
-    end;
-
-    [TestFixture]
-    TPipeIntegrationTests = class
-    private
-        FGraph: IGraphExecutor;
-        FLog: TCapturingLog;
-        FRootLayout: IScopeLayout;
-        FRootScope: IExecutionScope;
-        FInputChannel: IInputChannel;
-
-        // Simulates the Host Application setting up the environment ('btc' variable)
-        procedure SetupHostEnvironment;
-        // Compiles and Runs a script source
-        function CompileAndRun(const Source: string): IStream;
-    public
-        [Setup]
-        procedure Setup;
-        [Teardown]
-        procedure Teardown;
-
-        [Test]
-        procedure Test_Simple_Pipe_Throughput;
-
-        [Test]
-        procedure Test_Pipe_Field_Mapping_And_Calculation;
-    end;
-
-implementation
-
-{ TCapturingLog }
-
-constructor TCapturingLog.Create;
-begin
-    FErrors := TList<string>.Create;
-end;
-
-destructor TCapturingLog.Destroy;
-begin
-    FErrors.Free;
-    inherited;
-end;
-
-procedure TCapturingLog.Add(ALevel: TCompilerErrorLevel; const AMessage: string; const ANode: IAstNode);
-begin
-    if ALevel = elError then
-        FErrors.Add(AMessage);
-end;
-
-procedure TCapturingLog.AddError(const AMessage: string; const ANode: IAstNode);
-begin
-    Add(elError, AMessage, ANode);
-end;
-
-procedure TCapturingLog.AddWarning(const AMessage: string; const ANode: IAstNode);
-begin
-    // Ignore warnings for tests
-end;
-
-function TCapturingLog.GetEntries: TArray<TCompilerError>;
-begin
-    Result := nil;
-end;
-
-function TCapturingLog.GetEntryCount: Integer;
-begin
-    Result := FErrors.Count;
-end;
-
-function TCapturingLog.HasErrors: Boolean;
-begin
-    Result := FErrors.Count > 0;
-end;
-
-{ TDataCollector }
-
-constructor TDataCollector.Create(AStream: IStream);
-begin
-    FStream := AStream;
-    FSignalCount := 0;
-    FLastRecord := nil;
-end;
-
-procedure TDataCollector.OnSignal(const Signal: TStreamSignal);
-var
-    scalarVal: TScalar;
-begin
-    if Signal.Kind = skData then
-    begin
-        Inc(FSignalCount);
-        // Grab the latest item from the series
-        if FStream.Series.Count > 0 then
-        begin
-            scalarVal := FStream.Series.Items[FStream.Series.Count - 1];
-            // FIX: Cast TScalar to TDataValue to access AsScalarRecord
-            FLastRecord := TDataValue(scalarVal).AsScalarRecord;
-        end;
-    end;
-end;
-
-{ TPipeIntegrationTests }
-
-procedure TPipeIntegrationTests.Setup;
-begin
-    FGraph := TGraphExecutor.Create;
-    FLog := TCapturingLog.Create;
-    SetupHostEnvironment;
-end;
-
-procedure TPipeIntegrationTests.Teardown;
-begin
-    FGraph := nil; // Interfaces release automatically
-    FLog.Free; // Class implementation needs free (ref counting mixed usage)
-end;
-
-procedure TPipeIntegrationTests.SetupHostEnvironment;
-var
-    btcDef: IScalarRecordDefinition;
-    btcType: IStaticType;
-    typeDesc: IScopeDescriptor;
-    btcSlot: Integer;
-    scopeBuilder: IScopeBuilder;
-begin
-    // 1. Create the physical Input Channel in the Graph
-    // FIX: Use TKeywordRegistry.Intern for Keywords
-    // FIX: Using TScalarRecordDefinition class (check if renamed to TRecordDef in your codebase)
-    btcDef :=
-        TScalarRecordDefinition.Create(
-            [
-                TScalarRecordField.Create(TKeywordRegistry.Intern('Close'), skInt64),
-                TScalarRecordField.Create(TKeywordRegistry.Intern('Open'), skInt64)
-            ]
-        );
-    FInputChannel := FGraph.GetInputChannel('btc', btcDef);
-
-    // 2. Prepare the Compiler Environment (Symbol Table)
-    // FIX: Use Builder to define symbols
-    scopeBuilder := TScope.CreateBuilder(nil);
-    btcSlot := scopeBuilder.Define('btc');
-    FRootLayout := scopeBuilder.Build;
-
-    // 3. Prepare Type Information
-    // Tell the TypeChecker that 'btc' is a RecordSeries with Open/Close fields
-    btcType := TTypes.CreateRecordSeries(btcDef);
-    typeDesc := TScope.CreateDescriptor(FRootLayout, [btcType]);
-
-    // 4. Prepare Runtime Scope
-    // Create scope and inject the actual Stream object into slot 0
-    FRootScope := TScope.CreateScope(nil, typeDesc, nil);
-    FRootScope.DefineBoxed(btcSlot, TDataValue.FromStream(FInputChannel.Stream));
-end;
-
-function TPipeIntegrationTests.CompileAndRun(const Source: string): IStream;
-var
-    rawAst, boundAst, typedAst: IAstNode;
-    evaluator: IEvaluatorVisitor;
-    res: TDataValue;
-begin
-    // 1. Parse
-    try
-        rawAst := TAstScript.Parse(Source);
-    except
-        on E: Exception do
-            Assert.Fail('Parsing failed: ' + E.Message);
-    end;
-
-    // 2. Bind
-    boundAst := TAstBinder.Bind(FRootLayout, rawAst, FRootLayout, FLog);
-    if FLog.HasErrors then
-        Assert.Fail('Binding failed: ' + FLog.Errors[0]);
-
-    // 3. TypeCheck
-    typedAst := TTypeChecker.CheckTypes(boundAst, FRootLayout, FRootScope, FLog);
-    if FLog.HasErrors then
-        Assert.Fail('TypeCheck failed: ' + FLog.Errors[0]);
-
-    // 4. Evaluate
-    evaluator := TEvaluatorVisitor.Create(FRootScope);
-    res := evaluator.Execute(typedAst);
-
-    Assert.AreEqual(vkStream, res.Kind, 'Evaluation did not return a Stream');
-    Result := res.AsStream;
-end;
-
-procedure TPipeIntegrationTests.Test_Simple_Pipe_Throughput;
-var
-    pipeStream: IStream;
-    observer: TDataCollector;
-    inputData: IKeywordMapping<TScalar>;
-begin
-    // A simple pipe that just passes 'Close' through as 'Val'
-    var source := '(pipe [btc [:Close]] (fn [c] { :Val c }))';
-
-    pipeStream := CompileAndRun(source);
-
-    // Attach Observer
-    observer := TDataCollector.Create(pipeStream);
-    pipeStream.Subscribe(observer);
-
-    // --- Step 1: Input 100 ---
-    inputData := TKeywordMapping<TScalar>.Create([TPair<IKeyword, TScalar>.Create(TKeywordRegistry.Intern('Close'), 100)]);
-    FInputChannel.Push(inputData);
-
-    FGraph.Step;
-
-    // Assertions
-    Assert.AreEqual(1, observer.SignalCount, 'Should have received 1 signal');
-    Assert.IsNotNull(observer.LastRecord, 'Record should not be nil');
-    Assert.AreEqual(Int64(100), observer.LastRecord.Fields[TKeywordRegistry.Intern('Val')].Value.AsInt64, 'Output Value wrong');
-
-    // --- Step 2: Input 200 ---
-    inputData := TKeywordMapping<TScalar>.Create([TPair<IKeyword, TScalar>.Create(TKeywordRegistry.Intern('Close'), 200)]);
-    FInputChannel.Push(inputData);
-
-    FGraph.Step;
-
-    Assert.AreEqual(2, observer.SignalCount);
-    Assert.AreEqual(Int64(200), observer.LastRecord.Fields[TKeywordRegistry.Intern('Val')].Value.AsInt64);
-end;
-
-procedure TPipeIntegrationTests.Test_Pipe_Field_Mapping_And_Calculation;
-var
-    pipeStream: IStream;
-    observer: TDataCollector;
-    inputData: IKeywordMapping<TScalar>;
-begin
-    // A pipe that takes Open and Close, calculates the delta, and renames fields
-    // fn params: o -> Open, c -> Close
-    var source := '(pipe [btc [:Open :Close]] (fn [o c] { :Delta (- c o) :Original c }))';
-
-    pipeStream := CompileAndRun(source);
-    observer := TDataCollector.Create(pipeStream);
-    pipeStream.Subscribe(observer);
-
-    // Input: Open=100, Close=110 -> Delta should be 10
-    inputData :=
-        TKeywordMapping<TScalar>.Create(
-            [
-                TPair<IKeyword, TScalar>.Create(TKeywordRegistry.Intern('Open'), 100),
-                TPair<IKeyword, TScalar>.Create(TKeywordRegistry.Intern('Close'), 110)
-            ]
-        );
-    FInputChannel.Push(inputData);
-
-    FGraph.Step;
-
-    Assert.AreEqual(1, observer.SignalCount);
-
-    var delta := observer.LastRecord.Fields[TKeywordRegistry.Intern('Delta')].Value.AsInt64;
-    var orig := observer.LastRecord.Fields[TKeywordRegistry.Intern('Original')].Value.AsInt64;
-
-    Assert.AreEqual(Int64(10), delta, 'Delta calculation wrong');
-    Assert.AreEqual(Int64(110), orig, 'Original value passthrough wrong');
-end;
-
-initialization
-    TDUnitX.RegisterTestFixture(TPipeIntegrationTests);
-
-end.

Test/Demo.Finance.pas

@@ -11,7 +11,6 @@ uses
     Myc.Data.Scalar,
     Myc.Data.Keyword,
     Myc.Data.Stream, // IStream
-    Myc.Data.Stream.Pipes, // TRootStream
     Myc.Ast.Types,
     Myc.Ast.Scope,
     Myc.Ast;