Brummel/MycLib
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Pipe Optimization

Browse Source
This commit is contained in:
Michael Schimmel
2026-02-14 18:00:20 +01:00
parent dc46a2dd2d
commit cb8fd44d6f
3 changed files with 124 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Src/AST/Myc.Ast.Evaluator.pas
+1 -1
View File
@@ -636,7 +636,7 @@ begin
end;
//TODO Lookback not evaluated in script!
Result := TPipeStream.Create(config, outputDef, sources, pipeAdapter, 1000);
Result := TPipeStream.Create(config, outputDef, sources, pipeAdapter, 1000, 1);
end;
end.
Src/Data/Myc.Data.Stream.pas
+57 -50
View File
@@ -17,16 +17,17 @@ type
// TStreamSignal represents the data packet traveling through the graph.
// It carries the CycleID for synchronization and the actual data payload.
TStreamSignal = record
strict private
private
FCycleID: Int64;
FKind: TSignalKind;
FData: TArray<TScalar.TValue>;
FData: TScalar.PValue;
function GetFields(Idx: Integer): TScalar.TValue; inline;
function GetKind: TSignalKind; inline;
public
constructor Create(AKind: TSignalKind; ACycleID: Int64; const AData: TArray<TScalar.TValue> = nil);
constructor Create(ACycleID: Int64; const AData: TScalar.PValue);
function ToString: string;
property Fields[Idx: Integer]: TScalar.TValue read GetFields;
property CycleID: Int64 read FCycleID;
property Kind: TSignalKind read FKind;
property Data: TArray<TScalar.TValue> read FData;
property Kind: TSignalKind read GetKind;
end;
TSignalProc = reference to procedure(const Signal: TStreamSignal);
@@ -55,7 +56,7 @@ type
function GetDef: IScalarRecordDefinition;
protected
{ Broadcasts data to all subscribers. Stateless. }
procedure Emit(const Value: array of TScalar.TValue; ACycleID: Int64);
procedure Emit(CycleID: Int64; Values: TScalar.PValue);
public
constructor Create(const ADef: IScalarRecordDefinition);
destructor Destroy; override;
@@ -83,6 +84,7 @@ type
public
type
TSignalProc = reference to procedure(const Signal: TStreamSignal);
strict private
FSource: IStream;
FTag: TSubscriptionTag;
@@ -90,21 +92,21 @@ type
FFieldIndex: Integer;
FData: IWriteableSeries;
FLastSeenCycle: Int64;
FLookback: Int64;
FMaxLookback: Int64;
{ ISeries implementation }
function GetCount: Int64;
function GetItems(Idx: Integer): TScalar;
function GetTotalCount: Int64;
private
procedure HandleSignal(const Signal: TStreamSignal);
protected
private
procedure Detach;
public
constructor Create(const ASource: IStream; const AField: IKeyword; ALookback: Int64; const AOnSignal: TSignalProc);
constructor Create(const ASource: IStream; const AField: IKeyword; AMaxLookback: Int64; const AOnSignal: TSignalProc);
destructor Destroy; override;
function IsReady(RequiredLookback: Int64): Boolean;
property LastSeenCycle: Int64 read FLastSeenCycle;
end;
@@ -114,11 +116,13 @@ type
public
type
TPipeLambda = reference to function(const Sources: array of ISeries; out Results: array of TScalar.TValue): Boolean;
strict private
FSources: TArray<ISeries>;
FResults: TArray<TScalar.TValue>;
FLastFiredCycleID: Int64;
FLookback: Int64;
FMaxLookback: Int64;
FRequiredLookback: Int64;
FLambda: TPipeLambda;
private
@@ -130,27 +134,45 @@ type
const ADef: IScalarRecordDefinition;
const ASources: TArray<IStream>;
const ALambda: TPipeLambda;
ALookback: Int64
AMaxLookback, ARequiredLookback: Int64
);
destructor Destroy; override;
property Lookback: Int64 read FLookback;
property MaxLookback: Int64 read FMaxLookback;
property RequiredLookback: Int64 read FRequiredLookback;
end;
implementation
{ TStreamSignal }
constructor TStreamSignal.Create(AKind: TSignalKind; ACycleID: Int64; const AData: TArray<TScalar.TValue>);
constructor TStreamSignal.Create(ACycleID: Int64; const AData: TScalar.PValue);
begin
FKind := AKind;
FCycleID := ACycleID;
FData := AData;
end;
function TStreamSignal.GetFields(Idx: Integer): TScalar.TValue;
var
P: TScalar.PValue;
begin
Assert(FData <> nil, 'Heartbeat does not contain data');
P := FData;
inc(P, Idx);
Result := P^;
end;
function TStreamSignal.GetKind: TSignalKind;
begin
if FData <> nil then
Result := skData
else
Result := skHeartbeat;
end;
function TStreamSignal.ToString: string;
begin
if Kind = skData then
Result := Format('Signal(Data, #%d, Fields: %d)', [CycleID, Length(Data)])
Result := Format('Signal(Data, #%d)', [CycleID])
else
Result := Format('Signal(Heartbeat, #%d)', [CycleID]);
end;
@@ -197,19 +219,11 @@ begin
end;
end;
procedure TCustomDataStream.Emit(const Value: array of TScalar.TValue; ACycleID: Int64);
procedure TCustomDataStream.Emit(CycleID: Int64; Values: TScalar.PValue);
var
LSignal: TStreamSignal;
LData: TArray<TScalar.TValue>;
I: Integer;
begin
// Convert open array to dynamic array for signal payload
SetLength(LData, Length(Value));
for I := 0 to High(Value) do
LData[I] := Value[I];
LSignal := TStreamSignal.Create(skData, ACycleID, LData);
LSignal.Create(CycleID, Values);
FObservers.Lock;
try
FObservers.Notify(
@@ -220,6 +234,7 @@ begin
end
);
finally
LSignal.FData := nil;
FObservers.Release;
end;
end;
@@ -238,17 +253,17 @@ begin
raise EArgumentException.Create('Push: Data field count mismatch.');
Inc(FLastCycleID);
Emit(RowData, FLastCycleID);
Emit(FLastCycleID, @RowData[0]);
end;
{ TPipeSource }
constructor TPipeSource.Create(const ASource: IStream; const AField: IKeyword; ALookback: Int64; const AOnSignal: TSignalProc);
constructor TPipeSource.Create(const ASource: IStream; const AField: IKeyword; AMaxLookback: Int64; const AOnSignal: TSignalProc);
begin
inherited Create;
FSource := ASource;
FOnSignal := AOnSignal;
FLookback := ALookback;
FMaxLookback := AMaxLookback;
FLastSeenCycle := -1;
FFieldIndex := ASource.Def.IndexOf(AField);
@@ -278,17 +293,12 @@ begin
if (Signal.Kind = skData) then
begin
FLastSeenCycle := Signal.CycleID;
FData.Add(Signal.Data[FFieldIndex], FLookback);
FData.Add(Signal.Fields[FFieldIndex], FMaxLookback);
if Assigned(FOnSignal) then
FOnSignal(Signal);
end;
end;
function TPipeSource.IsReady(RequiredLookback: Int64): Boolean;
begin
Result := (FLastSeenCycle >= 0) and (FData.Count >= RequiredLookback);
end;
function TPipeSource.GetCount: Int64;
begin
Result := FData.Count;
@@ -311,14 +321,14 @@ constructor TPipeStream.Create(
const ADef: IScalarRecordDefinition;
const ASources: TArray<IStream>;
const ALambda: TPipeLambda;
ALookback: Int64
AMaxLookback, ARequiredLookback: Int64
);
var
i, j, n: Integer;
begin
inherited Create(ADef);
FLambda := ALambda;
FLookback := ALookback;
FMaxLookback := AMaxLookback;
FLastFiredCycleID := -1;
// Flatten sources from config
@@ -327,7 +337,7 @@ begin
inc(n, Length(AConfig[i]));
SetLength(FSources, n);
SetLength(FResults, n);
SetLength(FResults, ADef.Count);
n := 0;
for i := 0 to High(ASources) do
@@ -339,12 +349,13 @@ begin
TPipeSource.Create(
ASources[i],
AConfig[i][j].Key,
FLookback,
FMaxLookback,
procedure(const Signal: TStreamSignal) begin CheckBarrierAndFire(Signal.CycleID); end
);
inc(n);
end;
end;
FRequiredLookback := ARequiredLookback;
end;
destructor TPipeStream.Destroy;
@@ -356,12 +367,13 @@ end;
procedure TPipeStream.CheckBarrierAndFire(CurrentCycle: Int64);
begin
const requiredLookback = 10;
if FLastFiredCycleID >= CurrentCycle then
exit;
// 1. Check if all sources reached this cycle AND have enough history (Warm-up)
// Check if all sources reached this cycle AND have enough history (Warm-up)
for var src in FSources do
begin
if src.Count < requiredLookback then
if src.Count < FRequiredLookback then
exit;
var LSource := src as TPipeSource;
@@ -369,18 +381,13 @@ begin
exit;
end;
if FLastFiredCycleID >= CurrentCycle then
Exit;
FLastFiredCycleID := CurrentCycle;
if Assigned(FLambda) then
begin
// Execute transformation on synchronized window
if FLambda(FSources, FResults) then
begin
Emit(FResults, FLastFiredCycleID);
end;
Emit(FLastFiredCycleID, @FResults[0]);
end;
end;