Brummel/MycLib
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Pipes refactoring, Fixes in type checker

Browse Source
This commit is contained in:
Michael Schimmel
2026-02-13 23:34:39 +01:00
parent 19cdb2f004
commit dc46a2dd2d
10 changed files with 341 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Src/Data/Myc.Data.Stream.pas
+164 -100
View File
@@ -14,128 +14,143 @@ uses
type
TSignalKind = (skData, skHeartbeat);
// TStreamSignal represents the data packet traveling through the graph.
// It carries the CycleID for synchronization and the actual data payload.
TStreamSignal = record
private
strict private
FCycleID: Int64;
FKind: TSignalKind;
FData: TArray<TScalar.TValue>;
public
constructor Create(AKind: TSignalKind; ACycleID: Int64);
constructor Create(AKind: TSignalKind; ACycleID: Int64; const AData: TArray<TScalar.TValue> = nil);
function ToString: string;
property CycleID: Int64 read FCycleID;
property Kind: TSignalKind read FKind;
property Data: TArray<TScalar.TValue> read FData;
end;
IStreamObserver = interface
procedure OnSignal(const Signal: TStreamSignal);
end;
TSignalProc = reference to procedure(const Signal: TStreamSignal);
TSubscriptionTag = Pointer;
// IStream is now stateless. It defines WHAT is produced (Def), but not
// the history. History must be captured by an observer (Accumulator).
IStream = interface
{$region 'private'}
function GetSeries: IScalarRecordSeries;
function GetDef: IScalarRecordDefinition;
{$endregion}
function Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
function Subscribe(const Observer: TSignalProc): TSubscriptionTag;
procedure Unsubscribe(Tag: TSubscriptionTag);
// Returns the Series of the records this stream produces. Streams are scalar record series. Always!
property Series: IScalarRecordSeries read GetSeries;
{ Returns the definition (schema) of the records this stream produces. }
property Def: IScalarRecordDefinition read GetDef;
end;
// =========================================================================
// BASE STREAM
// Implements storage (Series) and broadcasting (Observers).
// Does not define WHEN data is emitted (Policy).
// =========================================================================
// Implements broadcasting logic. No internal storage of past values.
TCustomDataStream = class(TInterfacedObject, IStream)
strict private
FSeries: IWriteableScalarRecordSeries;
FObservers: TMycNotifyList<IStreamObserver>;
function GetSeries: IScalarRecordSeries;
FDef: IScalarRecordDefinition;
FObservers: TMycNotifyList<TSignalProc>;
function GetDef: IScalarRecordDefinition;
protected
// Derived classes call this to write data and notify listeners.
// ACycleID: The cycle this data belongs to.
{ Broadcasts data to all subscribers. Stateless. }
procedure Emit(const Value: array of TScalar.TValue; ACycleID: Int64);
public
constructor Create(const ADef: IScalarRecordDefinition);
destructor Destroy; override;
// IStream
function Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
function Subscribe(const Observer: TSignalProc): TSubscriptionTag;
procedure Unsubscribe(Tag: TSubscriptionTag);
property Series: IScalarRecordSeries read GetSeries;
property Def: IScalarRecordDefinition read GetDef;
end;
// =========================================================================
// ROOT STREAM (SOURCE)
// Acts as the clock source. Generates new CycleIDs.
// =========================================================================
// The entry point for external data. Generates the global CycleID (Clock).
TRootStream = class(TCustomDataStream)
private
strict private
FLastCycleID: Int64;
public
constructor Create(const ADef: IScalarRecordDefinition);
// Public API to inject data from Delphi
procedure Push(const RowData: TArray<TScalar.TValue>);
{ Injects a new record into the stream system. }
procedure Push(const RowData: array of TScalar.TValue);
end;
// =========================================================================
// PIPE STREAM (NODE)
// Reacts to upstream signals using barrier synchronization.
// =========================================================================
TPipeStream = class; // Forward
// Synchronizes multiple inputs via barrier and executes a transformation.
TPipeStream = class;
TPipeConfig = TArray<TArray<TScalarRecordField>>;
TPipeSource = class(TContainedObject, IStreamObserver)
private
{ TPipeSource acts as a selective accumulator for a single field of a stream. }
TPipeSource = class(TInterfacedObject, ISeries)
public
type
TSignalProc = reference to procedure(const Signal: TStreamSignal);
strict private
FSource: IStream;
FTag: TSubscriptionTag;
FOnSignal: TSignalProc;
FFieldIndex: Integer;
FData: IWriteableSeries;
FLastSeenCycle: Int64;
procedure OnSignal(const Signal: TStreamSignal);
FLookback: Int64;
{ ISeries implementation }
function GetCount: Int64;
function GetItems(Idx: Integer): TScalar;
function GetTotalCount: Int64;
private
procedure HandleSignal(const Signal: TStreamSignal);
protected
procedure Detach;
public
constructor Create(AOwner: TPipeStream; ASource: IStream);
constructor Create(const ASource: IStream; const AField: IKeyword; ALookback: Int64; const AOnSignal: TSignalProc);
destructor Destroy; override;
function IsReady(RequiredLookback: Int64): Boolean;
property LastSeenCycle: Int64 read FLastSeenCycle;
end;
TPipeConfig = TArray<TArray<TScalarRecordField>>;
TPipeStream = class(TCustomDataStream)
public
type
TPipeLambda = reference to function(const Sources: array of ISeries; out Results: array of TScalar.TValue): Boolean;
private
FSources: TArray<TPipeSource>;
FSourceSeries: TArray<ISeries>;
strict private
FSources: TArray<ISeries>;
FResults: TArray<TScalar.TValue>;
FLastFiredCycleID: Int64;
FLookback: Int64;
FLambda: TPipeLambda;
private
procedure CheckBarrierAndFire(CurrentCycle: Int64);
public
constructor Create(
const AConfig: TPipeConfig;
const ADef: IScalarRecordDefinition;
const ASources: TArray<IStream>;
const ALambda: TPipeLambda
const ALambda: TPipeLambda;
ALookback: Int64
);
destructor Destroy; override;
property Lookback: Int64 read FLookback;
end;
implementation
uses
Winapi.Windows;
{ TStreamSignal }
constructor TStreamSignal.Create(AKind: TSignalKind; ACycleID: Int64);
constructor TStreamSignal.Create(AKind: TSignalKind; ACycleID: Int64; const AData: TArray<TScalar.TValue>);
begin
FKind := AKind;
FCycleID := ACycleID;
FData := AData;
end;
function TStreamSignal.ToString: string;
begin
if Kind = skData then
Result := Format('Signal(Data, #%d)', [CycleID])
Result := Format('Signal(Data, #%d, Fields: %d)', [CycleID, Length(Data)])
else
Result := Format('Signal(Heartbeat, #%d)', [CycleID]);
end;
@@ -145,7 +160,7 @@ end;
constructor TCustomDataStream.Create(const ADef: IScalarRecordDefinition);
begin
inherited Create;
FSeries := TScalarRecordSeries.Create(ADef);
FDef := ADef;
end;
destructor TCustomDataStream.Destroy;
@@ -154,12 +169,12 @@ begin
inherited;
end;
function TCustomDataStream.GetSeries: IScalarRecordSeries;
function TCustomDataStream.GetDef: IScalarRecordDefinition;
begin
Result := FSeries;
Result := FDef;
end;
function TCustomDataStream.Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
function TCustomDataStream.Subscribe(const Observer: TSignalProc): TSubscriptionTag;
begin
FObservers.Lock;
try
@@ -171,6 +186,9 @@ end;
procedure TCustomDataStream.Unsubscribe(Tag: TSubscriptionTag);
begin
if Tag = nil then
exit;
FObservers.Lock;
try
FObservers.Unadvise(Tag);
@@ -181,20 +199,23 @@ end;
procedure TCustomDataStream.Emit(const Value: array of TScalar.TValue; ACycleID: Int64);
var
signal: TStreamSignal;
LSignal: TStreamSignal;
LData: TArray<TScalar.TValue>;
I: Integer;
begin
// Convert open array to dynamic array for signal payload
SetLength(LData, Length(Value));
for I := 0 to High(Value) do
LData[I] := Value[I];
LSignal := TStreamSignal.Create(skData, ACycleID, LData);
FObservers.Lock;
try
// 1. Write Data
FSeries.Add(Value);
// 2. Broadcast Signal
signal := TStreamSignal.Create(skData, ACycleID);
FObservers.Notify(
function(const Obs: IStreamObserver): Boolean
function(const Obs: TSignalProc): Boolean
begin
Obs.OnSignal(signal);
Obs(LSignal);
Result := True;
end
);
@@ -211,116 +232,159 @@ begin
FLastCycleID := 0;
end;
procedure TRootStream.Push(const RowData: TArray<TScalar.TValue>);
procedure TRootStream.Push(const RowData: array of TScalar.TValue);
begin
// Root streams increment the global clock
if Length(RowData) <> Def.Count then
raise EArgumentException.Create('Push: Data field count mismatch.');
Inc(FLastCycleID);
Emit(RowData, FLastCycleID);
end;
{ TPipeSource }
constructor TPipeSource.Create(AOwner: TPipeStream; ASource: IStream);
constructor TPipeSource.Create(const ASource: IStream; const AField: IKeyword; ALookback: Int64; const AOnSignal: TSignalProc);
begin
inherited Create(AOwner);
inherited Create;
FSource := ASource;
FOnSignal := AOnSignal;
FLookback := ALookback;
FLastSeenCycle := -1;
FTag := FSource.Subscribe(Self);
FFieldIndex := ASource.Def.IndexOf(AField);
if (FFieldIndex < 0) then
raise EArgumentException.CreateFmt('Field %s not found in source stream.', [AField.Name]);
FData := TScalarSeries.Create(ASource.Def.Items[FFieldIndex]);
FTag := FSource.Subscribe(procedure(const Signal: TStreamSignal) begin HandleSignal(Signal); end);
end;
destructor TPipeSource.Destroy;
begin
FSource.Unsubscribe(FTag);
Detach;
inherited;
end;
procedure TPipeSource.OnSignal(const Signal: TStreamSignal);
procedure TPipeSource.Detach;
begin
if Signal.Kind = skData then
FSource.Unsubscribe(FTag);
FTag := nil;
FOnSignal := nil;
end;
procedure TPipeSource.HandleSignal(const Signal: TStreamSignal);
begin
if (Signal.Kind = skData) then
begin
FLastSeenCycle := Signal.CycleID;
(Controller as TPipeStream).CheckBarrierAndFire(FLastSeenCycle);
FData.Add(Signal.Data[FFieldIndex], FLookback);
if Assigned(FOnSignal) then
FOnSignal(Signal);
end;
end;
function TPipeSource.IsReady(RequiredLookback: Int64): Boolean;
begin
Result := (FLastSeenCycle >= 0) and (FData.Count >= RequiredLookback);
end;
function TPipeSource.GetCount: Int64;
begin
Result := FData.Count;
end;
function TPipeSource.GetItems(Idx: Integer): TScalar;
begin
Result := FData.Items[Idx];
end;
function TPipeSource.GetTotalCount: Int64;
begin
Result := FData.TotalCount;
end;
{ TPipeStream }
constructor TPipeStream.Create(
const AConfig: TPipeConfig;
const ADef: IScalarRecordDefinition;
const ASources: TArray<IStream>;
const ALambda: TPipeLambda
const ALambda: TPipeLambda;
ALookback: Int64
);
var
i, j, n: Integer;
begin
// Pass Definition to base class
inherited Create(ADef);
FLambda := ALambda;
FLookback := ALookback;
FLastFiredCycleID := -1;
SetLength(FSources, Length(ASources));
// Flatten Sources
// Flatten sources from config
n := 0;
for i := 0 to High(AConfig) do
inc(n, Length(AConfig[i]));
SetLength(FSourceSeries, n);
SetLength(FSources, n);
SetLength(FResults, n);
n := 0;
for i := 0 to High(ASources) do
begin
FSources[i] := TPipeSource.Create(Self, ASources[i]);
for j := 0 to High(AConfig[i]) do
begin
// Extract the specific column series
var m := ASources[i].Series.IndexOf(AConfig[i][j].Key);
if m >= 0 then
begin
FSourceSeries[n] := ASources[i].Series[m];
inc(n);
end;
// Each input is wrapped in an accumulator source
FSources[n] :=
TPipeSource.Create(
ASources[i],
AConfig[i][j].Key,
FLookback,
procedure(const Signal: TStreamSignal) begin CheckBarrierAndFire(Signal.CycleID); end
);
inc(n);
end;
end;
end;
destructor TPipeStream.Destroy;
begin
for var i := High(FSources) downto 0 do
FSources[i].Free;
FSources := nil;
FSourceSeries := nil;
for var i := 0 to High(FSources) do
(FSources[i] as TPipeSource).Detach;
inherited;
end;
procedure TPipeStream.CheckBarrierAndFire(CurrentCycle: Int64);
var
resultVal: TArray<TScalar.TValue>;
begin
// We assume thread safety is handled by the caller
const requiredLookback = 10;
// 1. Check if all sources reached this cycle AND have enough history (Warm-up)
for var src in FSources do
if src.LastSeenCycle < CurrentCycle then
exit; // Barrier closed
begin
if src.Count < requiredLookback then
exit;
var LSource := src as TPipeSource;
if LSource.LastSeenCycle < CurrentCycle then
exit;
end;
if FLastFiredCycleID >= CurrentCycle then
exit; // Already fired
Exit;
FLastFiredCycleID := CurrentCycle;
if Assigned(FLambda) then
begin
SetLength(resultVal, Series.Def.Count);
if FLambda(FSourceSeries, resultVal) then
// Execute transformation on synchronized window
if FLambda(FSources, FResults) then
begin
// Pass through CycleID from upstream
Emit(resultVal, FLastFiredCycleID);
Emit(FResults, FLastFiredCycleID);
end;
end;
end;
initialization
TMycNotifyList<IStreamObserver>.ReverseOnNotify := False;
TMycNotifyList<TSignalProc>.ReverseOnNotify := False;
end.