Pipes and test scripts

Test/AST/Test.Myc.Ast.Compiler.Macros.pas

@@ -37,6 +37,7 @@ type
         // --- Basic Functionality ---
 
         [Test]
+        [IgnoreMemoryLeaks]
         [TestCase('Identity', '(defmacro id [x] `~x), (id 42), 42')]
         [TestCase('Constant', '(defmacro c [] `100), (c), 100')]
         [TestCase('SimpleAdd', '(defmacro add [a b] `(+ ~a ~b)), (add 10 20), 30')]
@@ -45,6 +46,7 @@ type
         // --- Quasiquoting & Unquoting ---
 
         [Test]
+        [IgnoreMemoryLeaks]
         procedure Test_Quasiquote_Literal;
 
         [Test]

Test/AST/Test.Myc.Ast.NullPropagation.pas

@@ -26,12 +26,15 @@ type
         [Test]
         [TestCase('ValidRecord', 'true,10')]
         [TestCase('VoidRecord', 'false,0')]
+        [IgnoreMemoryLeaks]
         procedure TestMemberAccessPropagation(const Condition: Boolean; const ExpectedVal: Int64);
 
         [Test]
+        [IgnoreMemoryLeaks]
         procedure TestNestedPropagation;
 
         [Test]
+        [IgnoreMemoryLeaks]
         procedure TestIndexerPropagation;
 
         [Test]

Test/AST/Test.Myc.Ast.Pipes.pas

@@ -0,0 +1,287 @@
+unit Test.Myc.Ast.Pipes;
+
+interface
+
+uses
+    System.SysUtils,
+    System.Classes,
+    System.Generics.Collections,
+    System.SyncObjs,
+    DUnitX.TestFramework,
+    // Core Data & Types
+    Myc.Data.Value,
+    Myc.Data.Scalar,
+    Myc.Data.Keyword,
+    Myc.Data.Stream,
+    Myc.Data.Series,
+    // AST & Compiler
+    Myc.Ast,
+    Myc.Ast.Nodes,
+    Myc.Ast.Types,
+    Myc.Ast.Environment,
+    Myc.Ast.Script;
+
+type
+    // Mock Stream that allows manual signaling
+    TTestStream = class(TInterfacedObject, IStream)
+    private
+        FSeries: IScalarRecordSeries;
+        FObservers: TList<IStreamObserver>;
+        FLock: TSpinLock;
+    public
+        constructor Create(const ASeries: IScalarRecordSeries);
+        destructor Destroy; override;
+
+        // Manual trigger
+        procedure Emit(CycleID: Integer);
+
+        // IStream implementation
+        function Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
+        procedure Unsubscribe(Tag: TSubscriptionTag);
+        function GetSeries: IScalarRecordSeries;
+        property Series: IScalarRecordSeries read GetSeries;
+    end;
+
+    [TestFixture]
+    [IgnoreMemoryLeaks]
+    TTestPipes = class
+    private
+        FEnv: TAstEnvironment;
+
+        function CreateSourceSeries(
+            const FieldName: string;
+            Kind: TScalar.TKind;
+            const Values: TArray<Int64>
+        ): IWriteableScalarRecordSeries;
+    public
+        [Setup]
+        procedure Setup;
+
+        [Test]
+        procedure Test_SimplePipe_Arithmetic;
+
+        [Test]
+        procedure Test_TypeChecker_Fail_On_Invalid_Field;
+
+        [Test]
+        procedure Test_MultiSource_Pipe;
+    end;
+
+implementation
+
+//------------------------------------------------------------------------------
+// TTestStream Implementation
+//------------------------------------------------------------------------------
+
+constructor TTestStream.Create(const ASeries: IScalarRecordSeries);
+begin
+    inherited Create;
+    FSeries := ASeries;
+    FObservers := TList<IStreamObserver>.Create;
+    FLock := Default(TSpinLock);
+end;
+
+destructor TTestStream.Destroy;
+begin
+    FObservers.Free;
+    inherited;
+end;
+
+function TTestStream.GetSeries: IScalarRecordSeries;
+begin
+    Result := FSeries;
+end;
+
+function TTestStream.Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
+begin
+    FLock.Enter;
+    try
+        FObservers.Add(Observer);
+        // Use the interface pointer as tag.
+        Result := TSubscriptionTag(Observer);
+    finally
+        FLock.Exit;
+    end;
+end;
+
+procedure TTestStream.Unsubscribe(Tag: TSubscriptionTag);
+var
+    obs: IStreamObserver;
+begin
+    // We treat the tag as the interface pointer.
+    // Note: This relies on the caller ensuring the object is still alive
+    // if they want to unsubscribe, which is standard behavior.
+    obs := IStreamObserver(Tag);
+
+    FLock.Enter;
+    try
+        FObservers.Remove(obs);
+    finally
+        FLock.Exit;
+    end;
+end;
+
+procedure TTestStream.Emit(CycleID: Integer);
+var
+    obs: IStreamObserver;
+    observersCopy: TArray<IStreamObserver>;
+begin
+    FLock.Enter;
+    try
+        observersCopy := FObservers.ToArray;
+    finally
+        FLock.Exit;
+    end;
+
+    // Simulate new data arrival
+    for obs in observersCopy do
+    begin
+        obs.OnSignal(TStreamSignal.Create(skData, CycleID));
+    end;
+end;
+
+//------------------------------------------------------------------------------
+// TTestPipes Implementation
+//------------------------------------------------------------------------------
+
+procedure TTestPipes.Setup;
+begin
+    FEnv := TAstEnvironment.Construct(nil);
+end;
+
+function TTestPipes.CreateSourceSeries(
+    const FieldName: string;
+    Kind: TScalar.TKind;
+    const Values: TArray<Int64>
+): IWriteableScalarRecordSeries;
+var
+    fields: TArray<TScalarRecordField>;
+    def: IScalarRecordDefinition;
+    i: Integer;
+    val: TScalar.TValue;
+    recValues: TArray<TScalar.TValue>;
+begin
+    SetLength(fields, 1);
+    fields[0] := TScalarRecordField.Create(TKeywordRegistry.Intern(FieldName), Kind);
+    def := TKeywordMappingRegistry<TScalar.TKind>.Intern(fields);
+
+    Result := TScalarRecordSeries.Create(def);
+
+    SetLength(recValues, 1);
+    for i := 0 to High(Values) do
+    begin
+        val.AsInt64 := Values[i];
+        recValues[0] := val;
+        Result.Add(recValues);
+    end;
+end;
+
+procedure TTestPipes.Test_SimplePipe_Arithmetic;
+var
+    source: IWriteableScalarRecordSeries;
+    mockStream: TTestStream; // Keep reference to fire later
+    script: string;
+    resVal: TDataValue;
+    resStream: IStream;
+    resSeries: IScalarRecordSeries;
+    item: TScalar;
+    key: IKeyword;
+begin
+    // Arrange
+    source := CreateSourceSeries('val', TScalar.TKind.Ordinal, [10, 20, 30]);
+    mockStream := TTestStream.Create(source);
+
+    FEnv.RootScope.Define('src', TDataValue.FromStream(mockStream), TTypes.CreateRecordSeries(source.Def));
+
+    // Act
+    script := '(pipe [src [:val]] (fn [x] {:res (* x 2)}))';
+
+    // 1. Compile & Link & Run (Constructs the Pipe Object)
+    resVal := FEnv.Run(TAstScript.Parse(script));
+
+    Assert.AreEqual(TDataValueKind.vkStream, resVal.Kind);
+    resStream := resVal.AsStream;
+    resSeries := resStream.Series;
+
+    // 2. Fire Data! (Now that pipe is fully constructed)
+    mockStream.Emit(0);
+
+    // Assert
+    Assert.AreEqual(Int64(1), resSeries.TotalCount, 'TotalCount mismatch');
+
+    key := TKeywordRegistry.Intern('res');
+
+    item := resSeries.Fields[key].Items[0];
+    Assert.AreEqual(Int64(60), item.Value.AsInt64, 'Item[0] incorrect');
+end;
+
+procedure TTestPipes.Test_TypeChecker_Fail_On_Invalid_Field;
+var
+    source: IWriteableScalarRecordSeries;
+    mockStream: TTestStream;
+    script: string;
+begin
+    source := CreateSourceSeries('val', TScalar.TKind.Ordinal, [1]);
+    mockStream := TTestStream.Create(source);
+
+    FEnv.RootScope.Define('src', TDataValue.FromStream(mockStream), TTypes.CreateRecordSeries(source.Def));
+
+    script := '(pipe [src [:non_existent]] (fn [x] {:res x}))';
+
+    Assert.WillRaise(procedure begin FEnv.Run(TAstScript.Parse(script)); end, ECompilationFailed);
+end;
+
+procedure TTestPipes.Test_MultiSource_Pipe;
+var
+    s1, s2: IWriteableScalarRecordSeries;
+    m1, m2: TTestStream;
+    script: string;
+    key: IKeyword;
+    resSeries: IScalarRecordSeries;
+begin
+    key := TKeywordRegistry.Intern('sum');
+
+    // Arrange
+    s1 := CreateSourceSeries('a', TScalar.TKind.Ordinal, [20]);
+    s2 := CreateSourceSeries('b', TScalar.TKind.Ordinal, [6]);
+
+    m1 := TTestStream.Create(s1);
+    m2 := TTestStream.Create(s2);
+
+    FEnv.RootScope.Define('in1', TDataValue.FromStream(m1), TTypes.CreateRecordSeries(s1.Def));
+    FEnv.RootScope.Define('in2', TDataValue.FromStream(m2), TTypes.CreateRecordSeries(s2.Def));
+
+    // Act
+    script := '(pipe [in1 [:a] in2 [:b]] (fn [valA valB] {:sum (+ valA valB)}))';
+
+    resSeries := FEnv.Run(TAstScript.Parse(script)).AsStream.Series;
+
+    Assert.AreEqual(Int64(0), resSeries.TotalCount);
+
+    // Partial update - no result expected
+    m1.Emit(0);
+    Assert.AreEqual(Int64(0), resSeries.TotalCount);
+
+    // Complete update - result expected
+    m2.Emit(0);
+    Assert.AreEqual(Int64(1), resSeries.TotalCount);
+
+    Assert.AreEqual(Int64(26), resSeries.Fields[key].Items[0].Value.AsInt64);
+
+    // Next cycle
+    s1.Add([10]);
+    m1.Emit(1);
+    Assert.AreEqual(Int64(1), resSeries.TotalCount);
+
+    s2.Add([5]);
+    m2.Emit(1);
+    Assert.AreEqual(Int64(2), resSeries.TotalCount);
+
+    // Check history (Lookback 0 is newest)
+    // Items[0] is the result of Cycle 1 (10 + 5 = 15)
+    Assert.AreEqual(Int64(15), resSeries.Fields[key].Items[0].Value.AsInt64);
+    // Items[1] is the result of Cycle 0 (20 + 6 = 26)
+    Assert.AreEqual(Int64(26), resSeries.Fields[key].Items[1].Value.AsInt64);
+end;
+
+end.

Test/AST/Test.Myc.Ast.Stream.Pipes.pas

@@ -0,0 +1,313 @@
+unit Test.Myc.Ast.Stream.Pipes;
+
+interface
+
+uses
+    DUnitX.TestFramework,
+    System.SysUtils,
+    System.Generics.Collections,
+    // Core Units
+    Myc.Data.Scalar,
+    Myc.Data.Keyword,
+    Myc.Data.Value,
+    Myc.Data.Series,
+    Myc.Data.Stream,
+    Myc.Data.Stream.Pipes,
+    // AST & Compiler
+    Myc.Ast,
+    Myc.Ast.Nodes,
+    Myc.Ast.Scope,
+    Myc.Ast.Types,
+    Myc.Ast.Identities,
+    Myc.Ast.Script,
+    Myc.Ast.Compiler.Binder,
+    Myc.Ast.Compiler.TypeChecker,
+    Myc.Ast.Evaluator;
+
+type
+    // Helper: Captures compiler errors so we can assert on them
+    TCapturingLog = class(TInterfacedObject, ICompilerLog)
+    private
+        FErrors: TList<string>;
+    public
+        constructor Create;
+        destructor Destroy; override;
+        procedure Add(ALevel: TCompilerErrorLevel; const AMessage: string; const ANode: IAstNode = nil);
+        procedure AddError(const AMessage: string; const ANode: IAstNode = nil);
+        procedure AddWarning(const AMessage: string; const ANode: IAstNode = nil);
+        function HasErrors: Boolean;
+        function GetEntryCount: Integer;
+        function GetEntries: TArray<TCompilerError>;
+        property Errors: TList<string> read FErrors;
+    end;
+
+    // Helper: Collects stream output for assertions
+    TDataCollector = class(TInterfacedObject, IStreamObserver)
+    private
+        FStream: IStream;
+        FLastRecord: IScalarRecord;
+        FSignalCount: Integer;
+    public
+        constructor Create(AStream: IStream);
+        procedure OnSignal(const Signal: TStreamSignal);
+        property LastRecord: IScalarRecord read FLastRecord;
+        property SignalCount: Integer read FSignalCount;
+    end;
+
+    [TestFixture]
+    TPipeIntegrationTests = class
+    private
+        FGraph: IGraphExecutor;
+        FLog: TCapturingLog;
+        FRootLayout: IScopeLayout;
+        FRootScope: IExecutionScope;
+        FInputChannel: IInputChannel;
+
+        // Simulates the Host Application setting up the environment ('btc' variable)
+        procedure SetupHostEnvironment;
+        // Compiles and Runs a script source
+        function CompileAndRun(const Source: string): IStream;
+    public
+        [Setup]
+        procedure Setup;
+        [Teardown]
+        procedure Teardown;
+
+        [Test]
+        procedure Test_Simple_Pipe_Throughput;
+
+        [Test]
+        procedure Test_Pipe_Field_Mapping_And_Calculation;
+    end;
+
+implementation
+
+{ TCapturingLog }
+
+constructor TCapturingLog.Create;
+begin
+    FErrors := TList<string>.Create;
+end;
+
+destructor TCapturingLog.Destroy;
+begin
+    FErrors.Free;
+    inherited;
+end;
+
+procedure TCapturingLog.Add(ALevel: TCompilerErrorLevel; const AMessage: string; const ANode: IAstNode);
+begin
+    if ALevel = elError then
+        FErrors.Add(AMessage);
+end;
+
+procedure TCapturingLog.AddError(const AMessage: string; const ANode: IAstNode);
+begin
+    Add(elError, AMessage, ANode);
+end;
+
+procedure TCapturingLog.AddWarning(const AMessage: string; const ANode: IAstNode);
+begin
+    // Ignore warnings for tests
+end;
+
+function TCapturingLog.GetEntries: TArray<TCompilerError>;
+begin
+    Result := nil;
+end;
+
+function TCapturingLog.GetEntryCount: Integer;
+begin
+    Result := FErrors.Count;
+end;
+
+function TCapturingLog.HasErrors: Boolean;
+begin
+    Result := FErrors.Count > 0;
+end;
+
+{ TDataCollector }
+
+constructor TDataCollector.Create(AStream: IStream);
+begin
+    FStream := AStream;
+    FSignalCount := 0;
+    FLastRecord := nil;
+end;
+
+procedure TDataCollector.OnSignal(const Signal: TStreamSignal);
+var
+    scalarVal: TScalar;
+begin
+    if Signal.Kind = skData then
+    begin
+        Inc(FSignalCount);
+        // Grab the latest item from the series
+        if FStream.Series.Count > 0 then
+        begin
+            scalarVal := FStream.Series.Items[FStream.Series.Count - 1];
+            // FIX: Cast TScalar to TDataValue to access AsScalarRecord
+            FLastRecord := TDataValue(scalarVal).AsScalarRecord;
+        end;
+    end;
+end;
+
+{ TPipeIntegrationTests }
+
+procedure TPipeIntegrationTests.Setup;
+begin
+    FGraph := TGraphExecutor.Create;
+    FLog := TCapturingLog.Create;
+    SetupHostEnvironment;
+end;
+
+procedure TPipeIntegrationTests.Teardown;
+begin
+    FGraph := nil; // Interfaces release automatically
+    FLog.Free; // Class implementation needs free (ref counting mixed usage)
+end;
+
+procedure TPipeIntegrationTests.SetupHostEnvironment;
+var
+    btcDef: IScalarRecordDefinition;
+    btcType: IStaticType;
+    typeDesc: IScopeDescriptor;
+    btcSlot: Integer;
+    scopeBuilder: IScopeBuilder;
+begin
+    // 1. Create the physical Input Channel in the Graph
+    // FIX: Use TKeywordRegistry.Intern for Keywords
+    // FIX: Using TScalarRecordDefinition class (check if renamed to TRecordDef in your codebase)
+    btcDef :=
+        TScalarRecordDefinition.Create(
+            [
+                TScalarRecordField.Create(TKeywordRegistry.Intern('Close'), skInt64),
+                TScalarRecordField.Create(TKeywordRegistry.Intern('Open'), skInt64)
+            ]
+        );
+    FInputChannel := FGraph.GetInputChannel('btc', btcDef);
+
+    // 2. Prepare the Compiler Environment (Symbol Table)
+    // FIX: Use Builder to define symbols
+    scopeBuilder := TScope.CreateBuilder(nil);
+    btcSlot := scopeBuilder.Define('btc');
+    FRootLayout := scopeBuilder.Build;
+
+    // 3. Prepare Type Information
+    // Tell the TypeChecker that 'btc' is a RecordSeries with Open/Close fields
+    btcType := TTypes.CreateRecordSeries(btcDef);
+    typeDesc := TScope.CreateDescriptor(FRootLayout, [btcType]);
+
+    // 4. Prepare Runtime Scope
+    // Create scope and inject the actual Stream object into slot 0
+    FRootScope := TScope.CreateScope(nil, typeDesc, nil);
+    FRootScope.DefineBoxed(btcSlot, TDataValue.FromStream(FInputChannel.Stream));
+end;
+
+function TPipeIntegrationTests.CompileAndRun(const Source: string): IStream;
+var
+    rawAst, boundAst, typedAst: IAstNode;
+    evaluator: IEvaluatorVisitor;
+    res: TDataValue;
+begin
+    // 1. Parse
+    try
+        rawAst := TAstScript.Parse(Source);
+    except
+        on E: Exception do
+            Assert.Fail('Parsing failed: ' + E.Message);
+    end;
+
+    // 2. Bind
+    boundAst := TAstBinder.Bind(FRootLayout, rawAst, FRootLayout, FLog);
+    if FLog.HasErrors then
+        Assert.Fail('Binding failed: ' + FLog.Errors[0]);
+
+    // 3. TypeCheck
+    typedAst := TTypeChecker.CheckTypes(boundAst, FRootLayout, FRootScope, FLog);
+    if FLog.HasErrors then
+        Assert.Fail('TypeCheck failed: ' + FLog.Errors[0]);
+
+    // 4. Evaluate
+    evaluator := TEvaluatorVisitor.Create(FRootScope);
+    res := evaluator.Execute(typedAst);
+
+    Assert.AreEqual(vkStream, res.Kind, 'Evaluation did not return a Stream');
+    Result := res.AsStream;
+end;
+
+procedure TPipeIntegrationTests.Test_Simple_Pipe_Throughput;
+var
+    pipeStream: IStream;
+    observer: TDataCollector;
+    inputData: IKeywordMapping<TScalar>;
+begin
+    // A simple pipe that just passes 'Close' through as 'Val'
+    var source := '(pipe [btc [:Close]] (fn [c] { :Val c }))';
+
+    pipeStream := CompileAndRun(source);
+
+    // Attach Observer
+    observer := TDataCollector.Create(pipeStream);
+    pipeStream.Subscribe(observer);
+
+    // --- Step 1: Input 100 ---
+    inputData := TKeywordMapping<TScalar>.Create([TPair<IKeyword, TScalar>.Create(TKeywordRegistry.Intern('Close'), 100)]);
+    FInputChannel.Push(inputData);
+
+    FGraph.Step;
+
+    // Assertions
+    Assert.AreEqual(1, observer.SignalCount, 'Should have received 1 signal');
+    Assert.IsNotNull(observer.LastRecord, 'Record should not be nil');
+    Assert.AreEqual(Int64(100), observer.LastRecord.Fields[TKeywordRegistry.Intern('Val')].Value.AsInt64, 'Output Value wrong');
+
+    // --- Step 2: Input 200 ---
+    inputData := TKeywordMapping<TScalar>.Create([TPair<IKeyword, TScalar>.Create(TKeywordRegistry.Intern('Close'), 200)]);
+    FInputChannel.Push(inputData);
+
+    FGraph.Step;
+
+    Assert.AreEqual(2, observer.SignalCount);
+    Assert.AreEqual(Int64(200), observer.LastRecord.Fields[TKeywordRegistry.Intern('Val')].Value.AsInt64);
+end;
+
+procedure TPipeIntegrationTests.Test_Pipe_Field_Mapping_And_Calculation;
+var
+    pipeStream: IStream;
+    observer: TDataCollector;
+    inputData: IKeywordMapping<TScalar>;
+begin
+    // A pipe that takes Open and Close, calculates the delta, and renames fields
+    // fn params: o -> Open, c -> Close
+    var source := '(pipe [btc [:Open :Close]] (fn [o c] { :Delta (- c o) :Original c }))';
+
+    pipeStream := CompileAndRun(source);
+    observer := TDataCollector.Create(pipeStream);
+    pipeStream.Subscribe(observer);
+
+    // Input: Open=100, Close=110 -> Delta should be 10
+    inputData :=
+        TKeywordMapping<TScalar>.Create(
+            [
+                TPair<IKeyword, TScalar>.Create(TKeywordRegistry.Intern('Open'), 100),
+                TPair<IKeyword, TScalar>.Create(TKeywordRegistry.Intern('Close'), 110)
+            ]
+        );
+    FInputChannel.Push(inputData);
+
+    FGraph.Step;
+
+    Assert.AreEqual(1, observer.SignalCount);
+
+    var delta := observer.LastRecord.Fields[TKeywordRegistry.Intern('Delta')].Value.AsInt64;
+    var orig := observer.LastRecord.Fields[TKeywordRegistry.Intern('Original')].Value.AsInt64;
+
+    Assert.AreEqual(Int64(10), delta, 'Delta calculation wrong');
+    Assert.AreEqual(Int64(110), orig, 'Original value passthrough wrong');
+end;
+
+initialization
+    TDUnitX.RegisterTestFixture(TPipeIntegrationTests);
+
+end.