Streams refactoring
This commit is contained in:
Michael Schimmel
File diff suppressed because one or more lines are too long
@@ -38,7 +38,6 @@ uses
|
||||
Myc.Fmx.AstEditor.Handlers.Data in '..\Src\AST\Myc.Fmx.AstEditor.Handlers.Data.pas',
|
||||
Myc.Ast.RTL.TypeRegistry in '..\Src\AST\Myc.Ast.RTL.TypeRegistry.pas',
|
||||
Myc.Trade.Broker in '..\Src\Myc.Trade.Broker.pas',
|
||||
Myc.Data.Stream.Pipes in '..\Src\Data\Myc.Data.Stream.Pipes.pas',
|
||||
Myc.Data.Stream in '..\Src\Data\Myc.Data.Stream.pas',
|
||||
Myc.Fmx.AstEditor.Handlers.Pipes in '..\Src\AST\Myc.Fmx.AstEditor.Handlers.Pipes.pas',
|
||||
Demo.Finance in '..\Test\Demo.Finance.pas',
|
||||
|
||||
@@ -168,7 +168,6 @@
|
||||
<DCCReference Include="..\Src\AST\Myc.Fmx.AstEditor.Handlers.Data.pas"/>
|
||||
<DCCReference Include="..\Src\AST\Myc.Ast.RTL.TypeRegistry.pas"/>
|
||||
<DCCReference Include="..\Src\Myc.Trade.Broker.pas"/>
|
||||
<DCCReference Include="..\Src\Data\Myc.Data.Stream.Pipes.pas"/>
|
||||
<DCCReference Include="..\Src\Data\Myc.Data.Stream.pas"/>
|
||||
<DCCReference Include="..\Src\AST\Myc.Fmx.AstEditor.Handlers.Pipes.pas"/>
|
||||
<DCCReference Include="..\Test\Demo.Finance.pas"/>
|
||||
|
||||
@@ -62,7 +62,6 @@ uses
|
||||
Myc.Data.Decimal,
|
||||
Myc.Data.Series,
|
||||
Myc.Data.Stream,
|
||||
Myc.Data.Stream.Pipes,
|
||||
Myc.Ast.Types;
|
||||
|
||||
type
|
||||
@@ -602,9 +601,7 @@ begin
|
||||
// Compile the transformation function
|
||||
lambdaFunc := Visit(N.Transformation).AsMethod();
|
||||
|
||||
// WARNING: If this is executing dynamically (without TypeChecker), StaticType is Unknown.
|
||||
// Pipe creation REQUIRES the output definition.
|
||||
// Fixed: Allow both stRecord and stRecordSeries (as TypeChecker correctly assigns stRecordSeries for Pipe nodes)
|
||||
// Allow both stRecord and stRecordSeries (as TypeChecker correctly assigns stRecordSeries for Pipe nodes)
|
||||
if (N.StaticType.Kind <> stRecordSeries) and (N.StaticType.Kind <> stRecord) then
|
||||
raise EEvaluatorException.Create('Pipe requires Type Checking to determine output structure (RecordDefinition).');
|
||||
|
||||
|
||||
@@ -1,280 +0,0 @@
|
||||
unit Myc.Data.Stream.Pipes;
|
||||
|
||||
interface
|
||||
|
||||
uses
|
||||
System.SysUtils,
|
||||
System.Classes,
|
||||
System.Generics.Collections,
|
||||
System.SyncObjs,
|
||||
Myc.Data.Scalar,
|
||||
Myc.Data.Keyword,
|
||||
Myc.Data.Stream,
|
||||
Myc.Core.Notifier;
|
||||
|
||||
type
|
||||
// =========================================================================
|
||||
// BASE STREAM
|
||||
// Implements storage (Series) and broadcasting (Observers).
|
||||
// Does not define WHEN data is emitted (Policy).
|
||||
// =========================================================================
|
||||
TCustomDataStream = class(TInterfacedObject, IStream)
|
||||
strict private
|
||||
FSeries: IWriteableScalarRecordSeries;
|
||||
FObservers: TMycNotifyList<IStreamObserver>;
|
||||
function GetSeries: IScalarRecordSeries;
|
||||
protected
|
||||
// Derived classes call this to write data and notify listeners.
|
||||
// ACycleID: The cycle this data belongs to.
|
||||
procedure Emit(const Value: array of TScalar.TValue; ACycleID: Int64);
|
||||
public
|
||||
constructor Create(const ADef: IScalarRecordDefinition);
|
||||
destructor Destroy; override;
|
||||
|
||||
// IStream
|
||||
function Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
|
||||
procedure Unsubscribe(Tag: TSubscriptionTag);
|
||||
property Series: IScalarRecordSeries read GetSeries;
|
||||
end;
|
||||
|
||||
// =========================================================================
|
||||
// ROOT STREAM (SOURCE)
|
||||
// Acts as the clock source. Generates new CycleIDs.
|
||||
// =========================================================================
|
||||
TRootStream = class(TCustomDataStream)
|
||||
private
|
||||
FLastCycleID: Int64;
|
||||
public
|
||||
constructor Create(const ADef: IScalarRecordDefinition);
|
||||
// Public API to inject data from Delphi
|
||||
procedure Push(const RowData: TArray<TScalar.TValue>);
|
||||
end;
|
||||
|
||||
// =========================================================================
|
||||
// PIPE STREAM (NODE)
|
||||
// Reacts to upstream signals using barrier synchronization.
|
||||
// =========================================================================
|
||||
TPipeStream = class; // Forward
|
||||
|
||||
TPipeConfig = TArray<TArray<TScalarRecordField>>;
|
||||
|
||||
TPipeSource = class(TContainedObject, IStreamObserver)
|
||||
private
|
||||
FSource: IStream;
|
||||
FTag: TSubscriptionTag;
|
||||
FLastSeenCycle: Int64;
|
||||
procedure OnSignal(const Signal: TStreamSignal);
|
||||
public
|
||||
constructor Create(AOwner: TPipeStream; ASource: IStream);
|
||||
destructor Destroy; override;
|
||||
property LastSeenCycle: Int64 read FLastSeenCycle;
|
||||
end;
|
||||
|
||||
TPipeStream = class(TCustomDataStream)
|
||||
public
|
||||
type
|
||||
TPipeLambda = reference to function(const Sources: array of ISeries; out Results: array of TScalar.TValue): Boolean;
|
||||
private
|
||||
FSources: TArray<TPipeSource>;
|
||||
FSourceSeries: TArray<ISeries>;
|
||||
FLastFiredCycleID: Int64;
|
||||
FLambda: TPipeLambda;
|
||||
|
||||
procedure CheckBarrierAndFire(CurrentCycle: Int64);
|
||||
public
|
||||
constructor Create(
|
||||
const AConfig: TPipeConfig;
|
||||
const ADef: IScalarRecordDefinition;
|
||||
const ASources: TArray<IStream>;
|
||||
const ALambda: TPipeLambda
|
||||
);
|
||||
destructor Destroy; override;
|
||||
end;
|
||||
|
||||
implementation
|
||||
|
||||
{ TCustomDataStream }
|
||||
|
||||
constructor TCustomDataStream.Create(const ADef: IScalarRecordDefinition);
|
||||
begin
|
||||
inherited Create;
|
||||
FSeries := TScalarRecordSeries.Create(ADef);
|
||||
end;
|
||||
|
||||
destructor TCustomDataStream.Destroy;
|
||||
begin
|
||||
FObservers.Finalize;
|
||||
inherited;
|
||||
end;
|
||||
|
||||
function TCustomDataStream.GetSeries: IScalarRecordSeries;
|
||||
begin
|
||||
Result := FSeries;
|
||||
end;
|
||||
|
||||
function TCustomDataStream.Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
|
||||
begin
|
||||
FObservers.Lock;
|
||||
try
|
||||
Result := FObservers.Advise(Observer);
|
||||
finally
|
||||
FObservers.Release;
|
||||
end;
|
||||
end;
|
||||
|
||||
procedure TCustomDataStream.Unsubscribe(Tag: TSubscriptionTag);
|
||||
begin
|
||||
FObservers.Lock;
|
||||
try
|
||||
FObservers.Unadvise(Tag);
|
||||
finally
|
||||
FObservers.Release;
|
||||
end;
|
||||
end;
|
||||
|
||||
procedure TCustomDataStream.Emit(const Value: array of TScalar.TValue; ACycleID: Int64);
|
||||
var
|
||||
signal: TStreamSignal;
|
||||
begin
|
||||
FObservers.Lock;
|
||||
try
|
||||
// 1. Write Data
|
||||
FSeries.Add(Value);
|
||||
|
||||
// 2. Broadcast Signal
|
||||
signal := TStreamSignal.Create(skData, ACycleID);
|
||||
|
||||
FObservers.Notify(
|
||||
function(const Obs: IStreamObserver): Boolean
|
||||
begin
|
||||
Obs.OnSignal(signal);
|
||||
Result := True;
|
||||
end
|
||||
);
|
||||
finally
|
||||
FObservers.Release;
|
||||
end;
|
||||
end;
|
||||
|
||||
{ TRootStream }
|
||||
|
||||
constructor TRootStream.Create(const ADef: IScalarRecordDefinition);
|
||||
begin
|
||||
inherited Create(ADef);
|
||||
FLastCycleID := 0;
|
||||
end;
|
||||
|
||||
procedure TRootStream.Push(const RowData: TArray<TScalar.TValue>);
|
||||
begin
|
||||
// Root streams increment the global clock
|
||||
Inc(FLastCycleID);
|
||||
Emit(RowData, FLastCycleID);
|
||||
end;
|
||||
|
||||
{ TPipeSource }
|
||||
|
||||
constructor TPipeSource.Create(AOwner: TPipeStream; ASource: IStream);
|
||||
begin
|
||||
inherited Create(AOwner);
|
||||
FSource := ASource;
|
||||
FLastSeenCycle := -1;
|
||||
FTag := FSource.Subscribe(Self);
|
||||
end;
|
||||
|
||||
destructor TPipeSource.Destroy;
|
||||
begin
|
||||
FSource.Unsubscribe(FTag);
|
||||
inherited;
|
||||
end;
|
||||
|
||||
procedure TPipeSource.OnSignal(const Signal: TStreamSignal);
|
||||
begin
|
||||
if Signal.Kind = skData then
|
||||
begin
|
||||
FLastSeenCycle := Signal.CycleID;
|
||||
(Controller as TPipeStream).CheckBarrierAndFire(FLastSeenCycle);
|
||||
end;
|
||||
end;
|
||||
|
||||
{ TPipeStream }
|
||||
|
||||
constructor TPipeStream.Create(
|
||||
const AConfig: TPipeConfig;
|
||||
const ADef: IScalarRecordDefinition;
|
||||
const ASources: TArray<IStream>;
|
||||
const ALambda: TPipeLambda
|
||||
);
|
||||
var
|
||||
i, j, n: Integer;
|
||||
begin
|
||||
// Pass Definition to base class
|
||||
inherited Create(ADef);
|
||||
|
||||
FLambda := ALambda;
|
||||
FLastFiredCycleID := -1;
|
||||
|
||||
SetLength(FSources, Length(ASources));
|
||||
|
||||
// Flatten Sources
|
||||
n := 0;
|
||||
for i := 0 to High(AConfig) do
|
||||
inc(n, Length(AConfig[i]));
|
||||
SetLength(FSourceSeries, n);
|
||||
|
||||
n := 0;
|
||||
for i := 0 to High(ASources) do
|
||||
begin
|
||||
FSources[i] := TPipeSource.Create(Self, ASources[i]);
|
||||
for j := 0 to High(AConfig[i]) do
|
||||
begin
|
||||
// Extract the specific column series
|
||||
FSourceSeries[n] := ASources[i].Series.Fields[AConfig[i][j].Key];
|
||||
inc(n);
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
destructor TPipeStream.Destroy;
|
||||
begin
|
||||
for var i := High(FSources) downto 0 do
|
||||
FSources[i].Free;
|
||||
FSources := nil;
|
||||
FSourceSeries := nil;
|
||||
inherited;
|
||||
end;
|
||||
|
||||
procedure TPipeStream.CheckBarrierAndFire(CurrentCycle: Int64);
|
||||
var
|
||||
resultVal: TArray<TScalar.TValue>;
|
||||
begin
|
||||
// We reuse the FObservers lock from the base class to protect state
|
||||
// But since FObservers is private, we access it via method or need to make it protected.
|
||||
// Making it protected or using a separate lock is cleaner.
|
||||
// For now, let's assume we rely on the fact that OnSignal is usually serialized per thread
|
||||
// or add a lock.
|
||||
// -> Ideally, TCustomDataStream should expose Lock/Unlock or we add a Lock here.
|
||||
|
||||
// Simplification: We assume thread safety is handled by the caller or we add a dedicated lock.
|
||||
// For this example, let's just do the logic:
|
||||
|
||||
for var src in FSources do
|
||||
if src.LastSeenCycle < CurrentCycle then
|
||||
exit; // Barrier closed
|
||||
|
||||
if FLastFiredCycleID >= CurrentCycle then
|
||||
exit; // Already fired
|
||||
|
||||
FLastFiredCycleID := CurrentCycle;
|
||||
|
||||
if Assigned(FLambda) then
|
||||
begin
|
||||
SetLength(resultVal, Series.Def.Count);
|
||||
if FLambda(FSourceSeries, resultVal) then
|
||||
begin
|
||||
// Pass through CycleID from upstream
|
||||
Emit(resultVal, FLastFiredCycleID);
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
end.
|
||||
+186
-123
@@ -43,61 +43,82 @@ type
|
||||
property Series: IScalarRecordSeries read GetSeries;
|
||||
end;
|
||||
|
||||
IInputChannel = interface
|
||||
{$region 'private'}
|
||||
function GetStream: IStream;
|
||||
{$endregion}
|
||||
procedure Push(const Value: IKeywordMapping<TScalar>);
|
||||
property Stream: IStream read GetStream;
|
||||
end;
|
||||
|
||||
IGraphExecutor = interface
|
||||
{$region 'private'}
|
||||
function GetCycleID: Int64;
|
||||
{$endregion}
|
||||
|
||||
function GetInputChannel(const Name: string; const Def: IScalarRecordDefinition): IInputChannel;
|
||||
procedure Step;
|
||||
property CycleID: Int64 read GetCycleID;
|
||||
end;
|
||||
|
||||
TGraphExecutor = class(TInterfacedObject, IGraphExecutor)
|
||||
private
|
||||
type
|
||||
TChannel = class(TInterfacedObject, IInputChannel, IStream)
|
||||
private
|
||||
type
|
||||
TStreamNotifier = TMycNotifyList<IStreamObserver>;
|
||||
private
|
||||
FName: string;
|
||||
FSeries: IWriteableScalarRecordSeries;
|
||||
FQueue: TQueue<IKeywordMapping<TScalar>>;
|
||||
FQueueLock: TSpinLock;
|
||||
FObservers: TStreamNotifier;
|
||||
|
||||
procedure Push(const Value: IKeywordMapping<TScalar>);
|
||||
function GetStream: IStream;
|
||||
function Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
|
||||
procedure Unsubscribe(Tag: TSubscriptionTag);
|
||||
procedure Emit(ACycle: Int64);
|
||||
|
||||
function GetSeries: IScalarRecordSeries;
|
||||
|
||||
public
|
||||
constructor Create(const AName: string; const ASeries: IWriteableScalarRecordSeries);
|
||||
destructor Destroy; override;
|
||||
end;
|
||||
|
||||
private
|
||||
FCycleID: Int64;
|
||||
FChannels: TDictionary<string, IInputChannel>;
|
||||
FLock: TSpinLock;
|
||||
function GetInputChannel(const Name: string; const Def: IScalarRecordDefinition): IInputChannel;
|
||||
function GetCycleID: Int64;
|
||||
// =========================================================================
|
||||
// BASE STREAM
|
||||
// Implements storage (Series) and broadcasting (Observers).
|
||||
// Does not define WHEN data is emitted (Policy).
|
||||
// =========================================================================
|
||||
TCustomDataStream = class(TInterfacedObject, IStream)
|
||||
strict private
|
||||
FSeries: IWriteableScalarRecordSeries;
|
||||
FObservers: TMycNotifyList<IStreamObserver>;
|
||||
function GetSeries: IScalarRecordSeries;
|
||||
protected
|
||||
// Derived classes call this to write data and notify listeners.
|
||||
// ACycleID: The cycle this data belongs to.
|
||||
procedure Emit(const Value: array of TScalar.TValue; ACycleID: Int64);
|
||||
public
|
||||
constructor Create;
|
||||
constructor Create(const ADef: IScalarRecordDefinition);
|
||||
destructor Destroy; override;
|
||||
|
||||
// IStream
|
||||
function Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
|
||||
procedure Unsubscribe(Tag: TSubscriptionTag);
|
||||
property Series: IScalarRecordSeries read GetSeries;
|
||||
end;
|
||||
|
||||
// =========================================================================
|
||||
// ROOT STREAM (SOURCE)
|
||||
// Acts as the clock source. Generates new CycleIDs.
|
||||
// =========================================================================
|
||||
TRootStream = class(TCustomDataStream)
|
||||
private
|
||||
FLastCycleID: Int64;
|
||||
public
|
||||
constructor Create(const ADef: IScalarRecordDefinition);
|
||||
// Public API to inject data from Delphi
|
||||
procedure Push(const RowData: TArray<TScalar.TValue>);
|
||||
end;
|
||||
|
||||
// =========================================================================
|
||||
// PIPE STREAM (NODE)
|
||||
// Reacts to upstream signals using barrier synchronization.
|
||||
// =========================================================================
|
||||
TPipeStream = class; // Forward
|
||||
|
||||
TPipeConfig = TArray<TArray<TScalarRecordField>>;
|
||||
|
||||
TPipeSource = class(TContainedObject, IStreamObserver)
|
||||
private
|
||||
FSource: IStream;
|
||||
FTag: TSubscriptionTag;
|
||||
FLastSeenCycle: Int64;
|
||||
procedure OnSignal(const Signal: TStreamSignal);
|
||||
public
|
||||
constructor Create(AOwner: TPipeStream; ASource: IStream);
|
||||
destructor Destroy; override;
|
||||
property LastSeenCycle: Int64 read FLastSeenCycle;
|
||||
end;
|
||||
|
||||
TPipeStream = class(TCustomDataStream)
|
||||
public
|
||||
type
|
||||
TPipeLambda = reference to function(const Sources: array of ISeries; out Results: array of TScalar.TValue): Boolean;
|
||||
private
|
||||
FSources: TArray<TPipeSource>;
|
||||
FSourceSeries: TArray<ISeries>;
|
||||
FLastFiredCycleID: Int64;
|
||||
FLambda: TPipeLambda;
|
||||
|
||||
procedure CheckBarrierAndFire(CurrentCycle: Int64);
|
||||
public
|
||||
constructor Create(
|
||||
const AConfig: TPipeConfig;
|
||||
const ADef: IScalarRecordDefinition;
|
||||
const ASources: TArray<IStream>;
|
||||
const ALambda: TPipeLambda
|
||||
);
|
||||
destructor Destroy; override;
|
||||
procedure Step;
|
||||
end;
|
||||
|
||||
implementation
|
||||
@@ -119,39 +140,26 @@ begin
|
||||
Result := Format('Signal(Heartbeat, #%d)', [CycleID]);
|
||||
end;
|
||||
|
||||
{ TGraphExecutor.TChannel }
|
||||
{ TCustomDataStream }
|
||||
|
||||
constructor TGraphExecutor.TChannel.Create(const AName: string; const ASeries: IWriteableScalarRecordSeries);
|
||||
constructor TCustomDataStream.Create(const ADef: IScalarRecordDefinition);
|
||||
begin
|
||||
inherited Create;
|
||||
FName := AName;
|
||||
FSeries := ASeries;
|
||||
FQueue := TQueue<IKeywordMapping<TScalar>>.Create;
|
||||
FSeries := TScalarRecordSeries.Create(ADef);
|
||||
end;
|
||||
|
||||
destructor TGraphExecutor.TChannel.Destroy;
|
||||
destructor TCustomDataStream.Destroy;
|
||||
begin
|
||||
FObservers.Finalize;
|
||||
FQueue.Free;
|
||||
inherited;
|
||||
end;
|
||||
|
||||
procedure TGraphExecutor.TChannel.Push(const Value: IKeywordMapping<TScalar>);
|
||||
function TCustomDataStream.GetSeries: IScalarRecordSeries;
|
||||
begin
|
||||
FQueueLock.Enter;
|
||||
try
|
||||
FQueue.Enqueue(Value);
|
||||
finally
|
||||
FQueueLock.Exit;
|
||||
end;
|
||||
Result := FSeries;
|
||||
end;
|
||||
|
||||
function TGraphExecutor.TChannel.GetStream: IStream;
|
||||
begin
|
||||
Result := Self;
|
||||
end;
|
||||
|
||||
function TGraphExecutor.TChannel.Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
|
||||
function TCustomDataStream.Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
|
||||
begin
|
||||
FObservers.Lock;
|
||||
try
|
||||
@@ -161,7 +169,7 @@ begin
|
||||
end;
|
||||
end;
|
||||
|
||||
procedure TGraphExecutor.TChannel.Unsubscribe(Tag: TSubscriptionTag);
|
||||
procedure TCustomDataStream.Unsubscribe(Tag: TSubscriptionTag);
|
||||
begin
|
||||
FObservers.Lock;
|
||||
try
|
||||
@@ -171,25 +179,18 @@ begin
|
||||
end;
|
||||
end;
|
||||
|
||||
procedure TGraphExecutor.TChannel.Emit(ACycle: Int64);
|
||||
procedure TCustomDataStream.Emit(const Value: array of TScalar.TValue; ACycleID: Int64);
|
||||
var
|
||||
signal: TStreamSignal;
|
||||
begin
|
||||
FQueueLock.Enter;
|
||||
try
|
||||
FSeries.Add(FQueue.Dequeue);
|
||||
signal :=
|
||||
TStreamSignal.Create(
|
||||
if FQueue.Count > 0 then skData
|
||||
else skHeartbeat,
|
||||
ACycle
|
||||
)
|
||||
finally
|
||||
FQueueLock.Exit;
|
||||
end;
|
||||
|
||||
FObservers.Lock;
|
||||
try
|
||||
// 1. Write Data
|
||||
FSeries.Add(Value);
|
||||
|
||||
// 2. Broadcast Signal
|
||||
signal := TStreamSignal.Create(skData, ACycleID);
|
||||
|
||||
FObservers.Notify(
|
||||
function(const Obs: IStreamObserver): Boolean
|
||||
begin
|
||||
@@ -202,63 +203,125 @@ begin
|
||||
end;
|
||||
end;
|
||||
|
||||
function TGraphExecutor.TChannel.GetSeries: IScalarRecordSeries;
|
||||
{ TRootStream }
|
||||
|
||||
constructor TRootStream.Create(const ADef: IScalarRecordDefinition);
|
||||
begin
|
||||
Result := FSeries;
|
||||
inherited Create(ADef);
|
||||
FLastCycleID := 0;
|
||||
end;
|
||||
|
||||
{ TGraphExecutor }
|
||||
|
||||
constructor TGraphExecutor.Create;
|
||||
procedure TRootStream.Push(const RowData: TArray<TScalar.TValue>);
|
||||
begin
|
||||
inherited Create;
|
||||
FChannels := TDictionary<string, IInputChannel>.Create;
|
||||
FCycleID := 0;
|
||||
// Root streams increment the global clock
|
||||
Inc(FLastCycleID);
|
||||
Emit(RowData, FLastCycleID);
|
||||
end;
|
||||
|
||||
destructor TGraphExecutor.Destroy;
|
||||
{ TPipeSource }
|
||||
|
||||
constructor TPipeSource.Create(AOwner: TPipeStream; ASource: IStream);
|
||||
begin
|
||||
FChannels.Free;
|
||||
inherited Create(AOwner);
|
||||
FSource := ASource;
|
||||
FLastSeenCycle := -1;
|
||||
FTag := FSource.Subscribe(Self);
|
||||
end;
|
||||
|
||||
destructor TPipeSource.Destroy;
|
||||
begin
|
||||
FSource.Unsubscribe(FTag);
|
||||
inherited;
|
||||
end;
|
||||
|
||||
function TGraphExecutor.GetInputChannel(const Name: string; const Def: IScalarRecordDefinition): IInputChannel;
|
||||
var
|
||||
impl: TChannel;
|
||||
procedure TPipeSource.OnSignal(const Signal: TStreamSignal);
|
||||
begin
|
||||
FLock.Enter;
|
||||
try
|
||||
if not FChannels.TryGetValue(Name, Result) then
|
||||
if Signal.Kind = skData then
|
||||
begin
|
||||
FLastSeenCycle := Signal.CycleID;
|
||||
(Controller as TPipeStream).CheckBarrierAndFire(FLastSeenCycle);
|
||||
end;
|
||||
end;
|
||||
|
||||
{ TPipeStream }
|
||||
|
||||
constructor TPipeStream.Create(
|
||||
const AConfig: TPipeConfig;
|
||||
const ADef: IScalarRecordDefinition;
|
||||
const ASources: TArray<IStream>;
|
||||
const ALambda: TPipeLambda
|
||||
);
|
||||
var
|
||||
i, j, n: Integer;
|
||||
begin
|
||||
// Pass Definition to base class
|
||||
inherited Create(ADef);
|
||||
|
||||
FLambda := ALambda;
|
||||
FLastFiredCycleID := -1;
|
||||
|
||||
SetLength(FSources, Length(ASources));
|
||||
|
||||
// Flatten Sources
|
||||
n := 0;
|
||||
for i := 0 to High(AConfig) do
|
||||
inc(n, Length(AConfig[i]));
|
||||
SetLength(FSourceSeries, n);
|
||||
|
||||
n := 0;
|
||||
for i := 0 to High(ASources) do
|
||||
begin
|
||||
FSources[i] := TPipeSource.Create(Self, ASources[i]);
|
||||
for j := 0 to High(AConfig[i]) do
|
||||
begin
|
||||
impl := TChannel.Create(Name, TScalarRecordSeries.Create(Def));
|
||||
Result := impl;
|
||||
FChannels.Add(Name, Result);
|
||||
// Extract the specific column series
|
||||
FSourceSeries[n] := ASources[i].Series.Fields[AConfig[i][j].Key];
|
||||
inc(n);
|
||||
end;
|
||||
finally
|
||||
FLock.Exit;
|
||||
end;
|
||||
end;
|
||||
|
||||
function TGraphExecutor.GetCycleID: Int64;
|
||||
destructor TPipeStream.Destroy;
|
||||
begin
|
||||
Result := FCycleID;
|
||||
for var i := High(FSources) downto 0 do
|
||||
FSources[i].Free;
|
||||
FSources := nil;
|
||||
FSourceSeries := nil;
|
||||
inherited;
|
||||
end;
|
||||
|
||||
procedure TGraphExecutor.Step;
|
||||
procedure TPipeStream.CheckBarrierAndFire(CurrentCycle: Int64);
|
||||
var
|
||||
channelsArr: TArray<IInputChannel>;
|
||||
channel: IInputChannel;
|
||||
resultVal: TArray<TScalar.TValue>;
|
||||
begin
|
||||
FLock.Enter;
|
||||
try
|
||||
Inc(FCycleID);
|
||||
channelsArr := FChannels.Values.ToArray;
|
||||
finally
|
||||
FLock.Exit;
|
||||
end;
|
||||
// We reuse the FObservers lock from the base class to protect state
|
||||
// But since FObservers is private, we access it via method or need to make it protected.
|
||||
// Making it protected or using a separate lock is cleaner.
|
||||
// For now, let's assume we rely on the fact that OnSignal is usually serialized per thread
|
||||
// or add a lock.
|
||||
// -> Ideally, TCustomDataStream should expose Lock/Unlock or we add a Lock here.
|
||||
|
||||
for channel in channelsArr do
|
||||
(channel as TChannel).Emit(FCycleID);
|
||||
// Simplification: We assume thread safety is handled by the caller or we add a dedicated lock.
|
||||
// For this example, let's just do the logic:
|
||||
|
||||
for var src in FSources do
|
||||
if src.LastSeenCycle < CurrentCycle then
|
||||
exit; // Barrier closed
|
||||
|
||||
if FLastFiredCycleID >= CurrentCycle then
|
||||
exit; // Already fired
|
||||
|
||||
FLastFiredCycleID := CurrentCycle;
|
||||
|
||||
if Assigned(FLambda) then
|
||||
begin
|
||||
SetLength(resultVal, Series.Def.Count);
|
||||
if FLambda(FSourceSeries, resultVal) then
|
||||
begin
|
||||
// Pass through CycleID from upstream
|
||||
Emit(resultVal, FLastFiredCycleID);
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
initialization
|
||||
|
||||
@@ -1,313 +0,0 @@
|
||||
unit Test.Myc.Ast.Stream.Pipes;
|
||||
|
||||
interface
|
||||
|
||||
uses
|
||||
DUnitX.TestFramework,
|
||||
System.SysUtils,
|
||||
System.Generics.Collections,
|
||||
// Core Units
|
||||
Myc.Data.Scalar,
|
||||
Myc.Data.Keyword,
|
||||
Myc.Data.Value,
|
||||
Myc.Data.Series,
|
||||
Myc.Data.Stream,
|
||||
Myc.Data.Stream.Pipes,
|
||||
// AST & Compiler
|
||||
Myc.Ast,
|
||||
Myc.Ast.Nodes,
|
||||
Myc.Ast.Scope,
|
||||
Myc.Ast.Types,
|
||||
Myc.Ast.Identities,
|
||||
Myc.Ast.Script,
|
||||
Myc.Ast.Compiler.Binder,
|
||||
Myc.Ast.Compiler.TypeChecker,
|
||||
Myc.Ast.Evaluator;
|
||||
|
||||
type
|
||||
// Helper: Captures compiler errors so we can assert on them
|
||||
TCapturingLog = class(TInterfacedObject, ICompilerLog)
|
||||
private
|
||||
FErrors: TList<string>;
|
||||
public
|
||||
constructor Create;
|
||||
destructor Destroy; override;
|
||||
procedure Add(ALevel: TCompilerErrorLevel; const AMessage: string; const ANode: IAstNode = nil);
|
||||
procedure AddError(const AMessage: string; const ANode: IAstNode = nil);
|
||||
procedure AddWarning(const AMessage: string; const ANode: IAstNode = nil);
|
||||
function HasErrors: Boolean;
|
||||
function GetEntryCount: Integer;
|
||||
function GetEntries: TArray<TCompilerError>;
|
||||
property Errors: TList<string> read FErrors;
|
||||
end;
|
||||
|
||||
// Helper: Collects stream output for assertions
|
||||
TDataCollector = class(TInterfacedObject, IStreamObserver)
|
||||
private
|
||||
FStream: IStream;
|
||||
FLastRecord: IScalarRecord;
|
||||
FSignalCount: Integer;
|
||||
public
|
||||
constructor Create(AStream: IStream);
|
||||
procedure OnSignal(const Signal: TStreamSignal);
|
||||
property LastRecord: IScalarRecord read FLastRecord;
|
||||
property SignalCount: Integer read FSignalCount;
|
||||
end;
|
||||
|
||||
[TestFixture]
|
||||
TPipeIntegrationTests = class
|
||||
private
|
||||
FGraph: IGraphExecutor;
|
||||
FLog: TCapturingLog;
|
||||
FRootLayout: IScopeLayout;
|
||||
FRootScope: IExecutionScope;
|
||||
FInputChannel: IInputChannel;
|
||||
|
||||
// Simulates the Host Application setting up the environment ('btc' variable)
|
||||
procedure SetupHostEnvironment;
|
||||
// Compiles and Runs a script source
|
||||
function CompileAndRun(const Source: string): IStream;
|
||||
public
|
||||
[Setup]
|
||||
procedure Setup;
|
||||
[Teardown]
|
||||
procedure Teardown;
|
||||
|
||||
[Test]
|
||||
procedure Test_Simple_Pipe_Throughput;
|
||||
|
||||
[Test]
|
||||
procedure Test_Pipe_Field_Mapping_And_Calculation;
|
||||
end;
|
||||
|
||||
implementation
|
||||
|
||||
{ TCapturingLog }
|
||||
|
||||
constructor TCapturingLog.Create;
|
||||
begin
|
||||
FErrors := TList<string>.Create;
|
||||
end;
|
||||
|
||||
destructor TCapturingLog.Destroy;
|
||||
begin
|
||||
FErrors.Free;
|
||||
inherited;
|
||||
end;
|
||||
|
||||
procedure TCapturingLog.Add(ALevel: TCompilerErrorLevel; const AMessage: string; const ANode: IAstNode);
|
||||
begin
|
||||
if ALevel = elError then
|
||||
FErrors.Add(AMessage);
|
||||
end;
|
||||
|
||||
procedure TCapturingLog.AddError(const AMessage: string; const ANode: IAstNode);
|
||||
begin
|
||||
Add(elError, AMessage, ANode);
|
||||
end;
|
||||
|
||||
procedure TCapturingLog.AddWarning(const AMessage: string; const ANode: IAstNode);
|
||||
begin
|
||||
// Ignore warnings for tests
|
||||
end;
|
||||
|
||||
function TCapturingLog.GetEntries: TArray<TCompilerError>;
|
||||
begin
|
||||
Result := nil;
|
||||
end;
|
||||
|
||||
function TCapturingLog.GetEntryCount: Integer;
|
||||
begin
|
||||
Result := FErrors.Count;
|
||||
end;
|
||||
|
||||
function TCapturingLog.HasErrors: Boolean;
|
||||
begin
|
||||
Result := FErrors.Count > 0;
|
||||
end;
|
||||
|
||||
{ TDataCollector }
|
||||
|
||||
constructor TDataCollector.Create(AStream: IStream);
|
||||
begin
|
||||
FStream := AStream;
|
||||
FSignalCount := 0;
|
||||
FLastRecord := nil;
|
||||
end;
|
||||
|
||||
procedure TDataCollector.OnSignal(const Signal: TStreamSignal);
|
||||
var
|
||||
scalarVal: TScalar;
|
||||
begin
|
||||
if Signal.Kind = skData then
|
||||
begin
|
||||
Inc(FSignalCount);
|
||||
// Grab the latest item from the series
|
||||
if FStream.Series.Count > 0 then
|
||||
begin
|
||||
scalarVal := FStream.Series.Items[FStream.Series.Count - 1];
|
||||
// FIX: Cast TScalar to TDataValue to access AsScalarRecord
|
||||
FLastRecord := TDataValue(scalarVal).AsScalarRecord;
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
{ TPipeIntegrationTests }
|
||||
|
||||
procedure TPipeIntegrationTests.Setup;
|
||||
begin
|
||||
FGraph := TGraphExecutor.Create;
|
||||
FLog := TCapturingLog.Create;
|
||||
SetupHostEnvironment;
|
||||
end;
|
||||
|
||||
procedure TPipeIntegrationTests.Teardown;
|
||||
begin
|
||||
FGraph := nil; // Interfaces release automatically
|
||||
FLog.Free; // Class implementation needs free (ref counting mixed usage)
|
||||
end;
|
||||
|
||||
procedure TPipeIntegrationTests.SetupHostEnvironment;
|
||||
var
|
||||
btcDef: IScalarRecordDefinition;
|
||||
btcType: IStaticType;
|
||||
typeDesc: IScopeDescriptor;
|
||||
btcSlot: Integer;
|
||||
scopeBuilder: IScopeBuilder;
|
||||
begin
|
||||
// 1. Create the physical Input Channel in the Graph
|
||||
// FIX: Use TKeywordRegistry.Intern for Keywords
|
||||
// FIX: Using TScalarRecordDefinition class (check if renamed to TRecordDef in your codebase)
|
||||
btcDef :=
|
||||
TScalarRecordDefinition.Create(
|
||||
[
|
||||
TScalarRecordField.Create(TKeywordRegistry.Intern('Close'), skInt64),
|
||||
TScalarRecordField.Create(TKeywordRegistry.Intern('Open'), skInt64)
|
||||
]
|
||||
);
|
||||
FInputChannel := FGraph.GetInputChannel('btc', btcDef);
|
||||
|
||||
// 2. Prepare the Compiler Environment (Symbol Table)
|
||||
// FIX: Use Builder to define symbols
|
||||
scopeBuilder := TScope.CreateBuilder(nil);
|
||||
btcSlot := scopeBuilder.Define('btc');
|
||||
FRootLayout := scopeBuilder.Build;
|
||||
|
||||
// 3. Prepare Type Information
|
||||
// Tell the TypeChecker that 'btc' is a RecordSeries with Open/Close fields
|
||||
btcType := TTypes.CreateRecordSeries(btcDef);
|
||||
typeDesc := TScope.CreateDescriptor(FRootLayout, [btcType]);
|
||||
|
||||
// 4. Prepare Runtime Scope
|
||||
// Create scope and inject the actual Stream object into slot 0
|
||||
FRootScope := TScope.CreateScope(nil, typeDesc, nil);
|
||||
FRootScope.DefineBoxed(btcSlot, TDataValue.FromStream(FInputChannel.Stream));
|
||||
end;
|
||||
|
||||
function TPipeIntegrationTests.CompileAndRun(const Source: string): IStream;
|
||||
var
|
||||
rawAst, boundAst, typedAst: IAstNode;
|
||||
evaluator: IEvaluatorVisitor;
|
||||
res: TDataValue;
|
||||
begin
|
||||
// 1. Parse
|
||||
try
|
||||
rawAst := TAstScript.Parse(Source);
|
||||
except
|
||||
on E: Exception do
|
||||
Assert.Fail('Parsing failed: ' + E.Message);
|
||||
end;
|
||||
|
||||
// 2. Bind
|
||||
boundAst := TAstBinder.Bind(FRootLayout, rawAst, FRootLayout, FLog);
|
||||
if FLog.HasErrors then
|
||||
Assert.Fail('Binding failed: ' + FLog.Errors[0]);
|
||||
|
||||
// 3. TypeCheck
|
||||
typedAst := TTypeChecker.CheckTypes(boundAst, FRootLayout, FRootScope, FLog);
|
||||
if FLog.HasErrors then
|
||||
Assert.Fail('TypeCheck failed: ' + FLog.Errors[0]);
|
||||
|
||||
// 4. Evaluate
|
||||
evaluator := TEvaluatorVisitor.Create(FRootScope);
|
||||
res := evaluator.Execute(typedAst);
|
||||
|
||||
Assert.AreEqual(vkStream, res.Kind, 'Evaluation did not return a Stream');
|
||||
Result := res.AsStream;
|
||||
end;
|
||||
|
||||
procedure TPipeIntegrationTests.Test_Simple_Pipe_Throughput;
|
||||
var
|
||||
pipeStream: IStream;
|
||||
observer: TDataCollector;
|
||||
inputData: IKeywordMapping<TScalar>;
|
||||
begin
|
||||
// A simple pipe that just passes 'Close' through as 'Val'
|
||||
var source := '(pipe [btc [:Close]] (fn [c] { :Val c }))';
|
||||
|
||||
pipeStream := CompileAndRun(source);
|
||||
|
||||
// Attach Observer
|
||||
observer := TDataCollector.Create(pipeStream);
|
||||
pipeStream.Subscribe(observer);
|
||||
|
||||
// --- Step 1: Input 100 ---
|
||||
inputData := TKeywordMapping<TScalar>.Create([TPair<IKeyword, TScalar>.Create(TKeywordRegistry.Intern('Close'), 100)]);
|
||||
FInputChannel.Push(inputData);
|
||||
|
||||
FGraph.Step;
|
||||
|
||||
// Assertions
|
||||
Assert.AreEqual(1, observer.SignalCount, 'Should have received 1 signal');
|
||||
Assert.IsNotNull(observer.LastRecord, 'Record should not be nil');
|
||||
Assert.AreEqual(Int64(100), observer.LastRecord.Fields[TKeywordRegistry.Intern('Val')].Value.AsInt64, 'Output Value wrong');
|
||||
|
||||
// --- Step 2: Input 200 ---
|
||||
inputData := TKeywordMapping<TScalar>.Create([TPair<IKeyword, TScalar>.Create(TKeywordRegistry.Intern('Close'), 200)]);
|
||||
FInputChannel.Push(inputData);
|
||||
|
||||
FGraph.Step;
|
||||
|
||||
Assert.AreEqual(2, observer.SignalCount);
|
||||
Assert.AreEqual(Int64(200), observer.LastRecord.Fields[TKeywordRegistry.Intern('Val')].Value.AsInt64);
|
||||
end;
|
||||
|
||||
procedure TPipeIntegrationTests.Test_Pipe_Field_Mapping_And_Calculation;
|
||||
var
|
||||
pipeStream: IStream;
|
||||
observer: TDataCollector;
|
||||
inputData: IKeywordMapping<TScalar>;
|
||||
begin
|
||||
// A pipe that takes Open and Close, calculates the delta, and renames fields
|
||||
// fn params: o -> Open, c -> Close
|
||||
var source := '(pipe [btc [:Open :Close]] (fn [o c] { :Delta (- c o) :Original c }))';
|
||||
|
||||
pipeStream := CompileAndRun(source);
|
||||
observer := TDataCollector.Create(pipeStream);
|
||||
pipeStream.Subscribe(observer);
|
||||
|
||||
// Input: Open=100, Close=110 -> Delta should be 10
|
||||
inputData :=
|
||||
TKeywordMapping<TScalar>.Create(
|
||||
[
|
||||
TPair<IKeyword, TScalar>.Create(TKeywordRegistry.Intern('Open'), 100),
|
||||
TPair<IKeyword, TScalar>.Create(TKeywordRegistry.Intern('Close'), 110)
|
||||
]
|
||||
);
|
||||
FInputChannel.Push(inputData);
|
||||
|
||||
FGraph.Step;
|
||||
|
||||
Assert.AreEqual(1, observer.SignalCount);
|
||||
|
||||
var delta := observer.LastRecord.Fields[TKeywordRegistry.Intern('Delta')].Value.AsInt64;
|
||||
var orig := observer.LastRecord.Fields[TKeywordRegistry.Intern('Original')].Value.AsInt64;
|
||||
|
||||
Assert.AreEqual(Int64(10), delta, 'Delta calculation wrong');
|
||||
Assert.AreEqual(Int64(110), orig, 'Original value passthrough wrong');
|
||||
end;
|
||||
|
||||
initialization
|
||||
TDUnitX.RegisterTestFixture(TPipeIntegrationTests);
|
||||
|
||||
end.
|
||||
@@ -11,7 +11,6 @@ uses
|
||||
Myc.Data.Scalar,
|
||||
Myc.Data.Keyword,
|
||||
Myc.Data.Stream, // IStream
|
||||
Myc.Data.Stream.Pipes, // TRootStream
|
||||
Myc.Ast.Types,
|
||||
Myc.Ast.Scope,
|
||||
Myc.Ast;
|
||||
|
||||
Reference in New Issue
Block a user