Adding Pipes

Src/Data/Myc.Data.Scalar.pas

@@ -133,21 +133,29 @@ type
     // A field definition for a scalar record.
     TScalarRecordField = TPair<IKeyword, TScalar.TKind>;
     IScalarRecordDefinition = IKeywordMapping<TScalar.TKind>;
-    IScalarRecord = IKeywordMapping<TScalar>;
 
-    TScalarRecord = class(TInterfacedObject, IScalarRecord)
+    IScalarRecord = interface(IKeywordMapping<TScalar>)
+        {$region 'private'}
+        function GetDef: IScalarRecordDefinition;
+        {$endregion}
+        property Def: IScalarRecordDefinition read GetDef;
+    end;
+
+    TScalarRecord = class(TInterfacedObject, IKeywordMapping<TScalar>, IScalarRecord)
         type
             TRegistry = TKeywordMappingRegistry<TScalar.TKind>;
     private
-        FDef: IScalarRecordDefinition;
         FData: TArray<TScalar.TValue>;
+        FDef: IScalarRecordDefinition;
 
         function IndexOf(const Key: IKeyword): Integer;
         function GetCount: Integer;
+        function GetDef: IScalarRecordDefinition;
         function GetItems(Idx: Integer): TPair<IKeyword, TScalar>;
         function GetFields(const Key: IKeyword): TScalar;
     public
         constructor Create(const ADef: IScalarRecordDefinition; const AData: TArray<TScalar.TValue>);
+        property Def: IScalarRecordDefinition read GetDef;
     end;
 
     ISeries = interface
@@ -179,27 +187,25 @@ type
         procedure Add(const Item: TScalar.TValue; Lookback: Int64 = -1);
     end;
 
-    IScalarRecordSeries = interface
+    IScalarRecordSeries = interface(IKeywordMapping<ISeries>)
         {$region 'private'}
-        function GetCount: Int64;
         function GetDef: IScalarRecordDefinition;
-        function GetFields(const Key: IKeyword): ISeries;
-        function GetTotalCount: Int64;
         {$endregion}
-        property Count: Int64 read GetCount;
         property Def: IScalarRecordDefinition read GetDef;
-        property Fields[const Key: IKeyword]: ISeries read GetFields; default;
-        property TotalCount: Int64 read GetTotalCount;
     end;
 
     IWriteableScalarRecordSeries = interface(IScalarRecordSeries)
         {$region 'private'}
+        function GetRecordCount: Int64;
+        function GetTotalCount: Int64;
         {$endregion}
-        procedure Add(const Item: IScalarRecord; Lookback: Int64 = -1);
+        procedure Add(const Item: IKeywordMapping<TScalar>; Lookback: Int64 = -1);
+        property RecordCount: Int64 read GetRecordCount;
+        property TotalCount: Int64 read GetTotalCount;
     end;
 
     // A series of scalar records, optimized for memory and access speed.
-    TScalarRecordSeries = class(TInterfacedObject, IWriteableScalarRecordSeries)
+    TScalarRecordSeries = class(TInterfacedObject, IWriteableScalarRecordSeries, IKeywordMapping<ISeries>)
         type
             TMemberSeries = class(TGenericContainedObject<TScalarRecordSeries>, ISeries)
             private
@@ -218,19 +224,24 @@ type
         FArray: TChunkArray<TScalar.TValue>;
         FFields: TArray<TMemberSeries>;
         FTotalCount: Int64;
-        function GetCount: Int64; inline;
+
+        function GetItems(Idx: Integer): TPair<IKeyword, ISeries>;
+        function GetCount: Integer;
+        function IndexOf(const Key: IKeyword): Integer;
+
+        function GetRecordCount: Int64; inline;
         function GetFields(const Key: IKeyword): ISeries;
         function GetDef: IScalarRecordDefinition; inline;
         function GetTotalCount: Int64; inline;
         function GetItemRef(Idx: Integer): TChunkArray<TScalar.TValue>.PT; inline;
+        function GetMapping: IKeywordMapping<ISeries>;
     public
         constructor Create(const ADef: IScalarRecordDefinition);
         destructor Destroy; override;
 
-        procedure Add(const Item: IScalarRecord; Lookback: Int64 = -1);
+        procedure Add(const Item: IKeywordMapping<TScalar>; Lookback: Int64 = -1);
 
-        property Count: Int64 read GetCount;
-        property Fields[const Key: IKeyword]: ISeries read GetFields; default;
+        property RecordCount: Int64 read GetRecordCount;
         property Def: IScalarRecordDefinition read GetDef;
         property TotalCount: Int64 read GetTotalCount;
     end;
@@ -803,7 +814,7 @@ begin
     inherited;
 end;
 
-procedure TScalarRecordSeries.Add(const Item: IScalarRecord; Lookback: Int64 = -1);
+procedure TScalarRecordSeries.Add(const Item: IKeywordMapping<TScalar>; Lookback: Int64 = -1);
 begin
     var lb := FDef.Count * Integer(Lookback);
     for var i := 0 to FDef.Count - 1 do
@@ -820,7 +831,7 @@ begin
     Result := FFields[elem];
 end;
 
-function TScalarRecordSeries.GetCount: Int64;
+function TScalarRecordSeries.GetRecordCount: Int64;
 begin
     Result := FArray.Count div FDef.Count;
 end;
@@ -836,11 +847,32 @@ begin
     Result := FArray.ItemRef[(FArray.Count - len) - (Idx * len)];
 end;
 
+function TScalarRecordSeries.GetMapping: IKeywordMapping<ISeries>;
+begin
+    Result := Self;
+end;
+
 function TScalarRecordSeries.GetTotalCount: Int64;
 begin
     Result := FTotalCount;
 end;
 
+function TScalarRecordSeries.GetCount: Integer;
+begin
+    Result := Length(FFields);
+end;
+
+function TScalarRecordSeries.GetItems(Idx: Integer): TPair<IKeyword, ISeries>;
+begin
+    Result.Key := FDef.Items[Idx].Key;
+    Result.Value := FFields[Idx];
+end;
+
+function TScalarRecordSeries.IndexOf(const Key: IKeyword): Integer;
+begin
+    Result := FDef.IndexOf(Key);
+end;
+
 { TScalar.TKindHelper }
 
 function TScalar.TKindHelper.ToString: string;
@@ -907,7 +939,7 @@ end;
 
 function TScalarRecordSeries.TMemberSeries.GetCount: Int64;
 begin
-    Result := RecordSeries.Count;
+    Result := RecordSeries.RecordCount;
 end;
 
 function TScalarRecordSeries.TMemberSeries.GetItems(Idx: Integer): TScalar;
@@ -998,6 +1030,11 @@ begin
     Result := FDef.Count;
 end;
 
+function TScalarRecord.GetDef: IScalarRecordDefinition;
+begin
+    Result := FDef;
+end;
+
 function TScalarRecord.GetFields(const Key: IKeyword): TScalar;
 var
     idx: Integer;

Src/Data/Myc.Data.Stream.pas

@@ -0,0 +1,268 @@
+unit Myc.Data.Stream;
+
+interface
+
+uses
+    System.SysUtils,
+    System.Classes,
+    System.Generics.Collections,
+    System.SyncObjs,
+    Myc.Data.Scalar,
+    Myc.Data.Value,
+    Myc.Core.Notifier;
+
+type
+    TSignalKind = (skData, skHeartbeat);
+
+    TStreamSignal = record
+    private
+        FCycleID: Int64;
+        FKind: TSignalKind;
+    public
+        constructor Create(AKind: TSignalKind; ACycleID: Int64);
+        function ToString: string;
+        property CycleID: Int64 read FCycleID;
+        property Kind: TSignalKind read FKind;
+    end;
+
+    IStreamObserver = interface
+        procedure OnSignal(const Signal: TStreamSignal);
+    end;
+
+    TSubscriptionTag = Pointer;
+
+    IStream = interface
+        {$region 'private'}
+        function GetSeries: IScalarRecordSeries;
+        {$endregion}
+
+        function Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
+        procedure Unsubscribe(Tag: TSubscriptionTag);
+
+        // Returns the Series of the records this stream produces. Streams are scalar record series. Always!
+        property Series: IScalarRecordSeries read GetSeries;
+    end;
+
+    IInputChannel = interface
+        {$region 'private'}
+        function GetStream: IStream;
+        {$endregion}
+
+        procedure Push(const Value: IScalarRecord);
+        property Stream: IStream read GetStream;
+    end;
+
+    IGraphExecutor = interface
+        {$region 'private'}
+        function GetCycleID: Int64;
+        {$endregion}
+
+        // Now requires a Definition when creating a channel
+        function GetInputChannel(const Name: string; const Def: IScalarRecordDefinition): IInputChannel;
+        procedure Step;
+        property CycleID: Int64 read GetCycleID;
+    end;
+
+    TGraphExecutor = class(TInterfacedObject, IGraphExecutor)
+    private
+        type
+            TChannel = class(TInterfacedObject, IInputChannel, IStream)
+            private
+                type
+                    TStreamNotifier = TMycNotifyList<IStreamObserver>;
+            private
+                FName: string;
+                FSeries: IWriteableScalarRecordSeries;
+                FQueue: TQueue<IScalarRecord>;
+                FQueueLock: TSpinLock;
+                FObservers: TStreamNotifier;
+
+                procedure Push(const Value: IScalarRecord);
+                function GetStream: IStream;
+                function Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
+                procedure Unsubscribe(Tag: TSubscriptionTag);
+                procedure Emit(ACycle: Int64);
+
+                function GetSeries: IScalarRecordSeries;
+
+            public
+                constructor Create(const AName: string; const ASeries: IWriteableScalarRecordSeries);
+                destructor Destroy; override;
+            end;
+
+    private
+        FCycleID: Int64;
+        FChannels: TDictionary<string, IInputChannel>;
+        FLock: TSpinLock;
+        function GetInputChannel(const Name: string; const Def: IScalarRecordDefinition): IInputChannel;
+        function GetCycleID: Int64;
+    public
+        constructor Create;
+        destructor Destroy; override;
+        procedure Step;
+    end;
+
+implementation
+
+constructor TStreamSignal.Create(AKind: TSignalKind; ACycleID: Int64);
+begin
+    FKind := AKind;
+    FCycleID := ACycleID;
+end;
+
+function TStreamSignal.ToString: string;
+begin
+    if Kind = skData then
+        Result := Format('Signal(Data, #%d)', [CycleID])
+    else
+        Result := Format('Signal(Heartbeat, #%d)', [CycleID]);
+end;
+
+{ TGraphExecutor.TChannel }
+
+constructor TGraphExecutor.TChannel.Create(const AName: string; const ASeries: IWriteableScalarRecordSeries);
+begin
+    inherited Create;
+    FName := AName;
+    FSeries := ASeries;
+    FQueue := TQueue<IScalarRecord>.Create;
+end;
+
+destructor TGraphExecutor.TChannel.Destroy;
+begin
+    FObservers.Finalize;
+    FQueue.Free;
+    inherited;
+end;
+
+procedure TGraphExecutor.TChannel.Push(const Value: IScalarRecord);
+begin
+    Assert(Value.Def = FSeries.Def);
+
+    FQueueLock.Enter;
+    try
+        FQueue.Enqueue(Value);
+    finally
+        FQueueLock.Exit;
+    end;
+end;
+
+function TGraphExecutor.TChannel.GetStream: IStream;
+begin
+    Result := Self;
+end;
+
+function TGraphExecutor.TChannel.Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
+begin
+    FObservers.Lock;
+    try
+        Result := FObservers.Advise(Observer);
+    finally
+        FObservers.Release;
+    end;
+end;
+
+procedure TGraphExecutor.TChannel.Unsubscribe(Tag: TSubscriptionTag);
+begin
+    FObservers.Lock;
+    try
+        FObservers.Unadvise(Tag);
+    finally
+        FObservers.Release;
+    end;
+end;
+
+procedure TGraphExecutor.TChannel.Emit(ACycle: Int64);
+var
+    signal: TStreamSignal;
+begin
+    FQueueLock.Enter;
+    try
+        FSeries.Add(FQueue.Dequeue);
+        signal :=
+            TStreamSignal.Create(
+                if FQueue.Count > 0 then skData
+                else skHeartbeat,
+                ACycle
+            )
+    finally
+        FQueueLock.Exit;
+    end;
+
+    FObservers.Lock;
+    try
+        FObservers.Notify(
+            function(const Obs: IStreamObserver): Boolean
+            begin
+                Obs.OnSignal(signal);
+                Result := True;
+            end
+        );
+    finally
+        FObservers.Release;
+    end;
+end;
+
+function TGraphExecutor.TChannel.GetSeries: IScalarRecordSeries;
+begin
+    Result := FSeries;
+end;
+
+{ TGraphExecutor }
+
+constructor TGraphExecutor.Create;
+begin
+    inherited Create;
+    FChannels := TDictionary<string, IInputChannel>.Create;
+    FCycleID := 0;
+end;
+
+destructor TGraphExecutor.Destroy;
+begin
+    FChannels.Free;
+    inherited;
+end;
+
+function TGraphExecutor.GetInputChannel(const Name: string; const Def: IScalarRecordDefinition): IInputChannel;
+var
+    impl: TChannel;
+begin
+    FLock.Enter;
+    try
+        if not FChannels.TryGetValue(Name, Result) then
+        begin
+            impl := TChannel.Create(Name, TScalarRecordSeries.Create(Def));
+            Result := impl;
+            FChannels.Add(Name, Result);
+        end;
+    finally
+        FLock.Exit;
+    end;
+end;
+
+function TGraphExecutor.GetCycleID: Int64;
+begin
+    Result := FCycleID;
+end;
+
+procedure TGraphExecutor.Step;
+var
+    channelsArr: TArray<IInputChannel>;
+    channel: IInputChannel;
+begin
+    FLock.Enter;
+    try
+        Inc(FCycleID);
+        channelsArr := FChannels.Values.ToArray;
+    finally
+        FLock.Exit;
+    end;
+
+    for channel in channelsArr do
+        (channel as TChannel).Emit(FCycleID);
+end;
+
+initialization
+    TMycNotifyList<IStreamObserver>.ReverseOnNotify := False;
+
+end.

Src/Data/Myc.Data.Value.pas

@@ -397,7 +397,7 @@ begin
                     first := False;
                 end;
                 sb.Append('}');
-                Result := Format('<record_series%s[%d]>', [sb.ToString, series.Count]);
+                Result := Format('<record_series%s[%d]>', [sb.ToString, series.RecordCount]);
             finally
                 sb.Free;
             end;