Brummel/MycLib
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Pipes refactoring, Fixes in type checker

Browse Source
This commit is contained in:
Michael Schimmel
2026-02-13 23:34:39 +01:00
parent 19cdb2f004
commit dc46a2dd2d
10 changed files with 341 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Src/AST/Myc.Ast.Compiler.TypeChecker.pas
+40 -17
View File
@@ -629,16 +629,36 @@ begin
if Length(sig.ParamTypes) <> Length(argTypes) then
continue;
match := True;
var sigIsStatic := sig.ReturnType.Kind <> stUnknown;
for j := 0 to High(argTypes) do
begin
if not TTypeRules.CanAssign(sig.ParamTypes[j], argTypes[j]) then
begin
match := False;
break;
end;
if sig.ParamTypes[j].Kind = stUnknown then
sigIsStatic := false;
end;
if match then
begin
bestSig := sig;
break;
if not sigIsStatic then
begin
if bestSig <> nil then
if Assigned(FLog) then
FLog.AddError(
Format('RTL contains ambiguous signatures for method call on %s', [calleeType.ToString]),
Node
);
bestSig := sig;
end
else
begin
bestSig := sig;
break;
end;
end;
end;
@@ -763,19 +783,21 @@ begin
baseType := PrepareBaseType(newBase, isOpt);
resType := TTypes.Unknown;
if (baseType.Kind = stRecord) or (baseType.Kind = stRecordSeries) then
begin
idx := baseType.AsRecord.Definition.IndexOf(M.Member.Value);
if idx >= 0 then
case baseType.Kind of
stRecord, stRecordSeries:
begin
var fieldType := TTypes.FromScalarKind(baseType.AsRecord.Definition[idx]);
if baseType.Kind = stRecordSeries then
resType := TTypes.CreateSeries(fieldType)
else
resType := fieldType;
end
else if Assigned(FLog) then
FLog.AddError('Member not found', Node);
idx := baseType.AsRecord.Definition.IndexOf(M.Member.Value);
if idx >= 0 then
begin
var fieldType := TTypes.FromScalarKind(baseType.AsRecord.Definition[idx]);
if baseType.Kind = stRecordSeries then
resType := TTypes.CreateSeries(fieldType)
else
resType := fieldType;
end
else if Assigned(FLog) then
FLog.AddError('Member not found', Node);
end;
end;
resType := ApplyOptionality(resType, isOpt);
@@ -1081,7 +1103,8 @@ begin
else
inferredType := TTypes.Ordinal;
end;
paramTypes.Add(inferredType);
paramTypes.Add(TTypes.CreateSeries(inferredType));
end;
// Reconstruct the typed Input Tuple [StreamIdent, [Selectors]]
@@ -1150,7 +1173,7 @@ begin
end
else
begin
if (lambdaRetType.Kind <> stUnknown) and (lambdaRetType.Kind <> stVoid) then
if lambdaRetType.Kind <> stVoid then
begin
if Assigned(FLog) then
FLog.AddError(
@@ -1158,7 +1181,7 @@ begin
lambda
);
end;
pipeType := TTypes.Unknown;
pipeType := TTypes.Void;
end;
// Reconstruct the Pipe Node with typed Inputs tuple and typed Lambda
Src/AST/Myc.Ast.Evaluator.pas
+15 -9
View File
@@ -411,7 +411,7 @@ begin
vkRecordSeries: Result := base.AsRecordSeries.Fields[N.Member.Value];
vkScalarRecord: Result := base.AsScalarRecord.Fields[N.Member.Value];
vkRecord: Result := base.AsRecord.Fields[N.Member.Value];
vkStream: Result := base.AsStream.Series.Fields[N.Member.Value];
// vkStream: Result := base.AsStream.Def.Fields[N.Member.Value];
else
raise EEvaluatorException.Create('Member error');
end;
@@ -549,7 +549,6 @@ var
sources: TArray<IStream>;
config: TPipeConfig;
inputVal: TDataValue;
sourceSeries: IScalarRecordSeries;
lambdaFunc: TDataValue.TFunc;
inputsElements: TArray<IAstNode>;
@@ -581,8 +580,7 @@ begin
raise EEvaluatorException.Create(Format('Variable "%s" is not a Stream.', [sourceId.Name]));
sources[i] := inputVal.AsStream;
sourceSeries := sources[i].Series;
var srcDef := sourceSeries.Def;
var srcDef := sources[i].Def;
// Build Selectors Config
SetLength(config[i], Length(selectorsElements));
@@ -601,11 +599,18 @@ begin
// Compile the transformation function
lambdaFunc := Visit(N.Transformation).AsMethod();
// Allow both stRecord and stRecordSeries (as TypeChecker correctly assigns stRecordSeries for Pipe nodes)
if (N.StaticType.Kind <> stRecordSeries) and (N.StaticType.Kind <> stRecord) then
raise EEvaluatorException.Create('Pipe requires Type Checking to determine output structure (RecordDefinition).');
var outputDef: IScalarRecordDefinition;
var outputDef := N.StaticType.AsRecord.Definition;
case N.StaticType.Kind of
stVoid:
// This is an endpoint. The lambda returns nothing. So we produce "nothing".
outputDef := TScalarRecord.TRegistry.Empty;
stRecord, stRecordSeries:
// Allow both stRecord and stRecordSeries (as TypeChecker correctly assigns stRecordSeries for Pipe nodes)
outputDef := N.StaticType.AsRecord.Definition;
else
raise EEvaluatorException.Create('Pipe requires Type Checking to determine output structure (RecordDefinition).');
end;
var pipeAdapter: TPipeStream.TPipeLambda :=
function(const S: array of ISeries; out R: array of TScalar.TValue): Boolean
@@ -630,7 +635,8 @@ begin
Result := True;
end;
Result := TPipeStream.Create(config, outputDef, sources, pipeAdapter);
//TODO Lookback not evaluated in script!
Result := TPipeStream.Create(config, outputDef, sources, pipeAdapter, 1000);
end;
end.
Src/Data/Myc.Data.Keyword.pas
+4
View File
@@ -86,12 +86,14 @@ type
class var
FLock: TSpinLock;
FRegistry: TDictionary<TArray<Integer>, IKeywordMapping<T>>;
FEmpty: IKeywordMapping<T>;
class constructor Create;
class destructor Destroy;
public
// Creates or gets the cached keyword mapping.
class function Intern(const AFields: TArray<TPair<IKeyword, T>>): IKeywordMapping<T>; static;
class property Empty: IKeywordMapping<T> read FEmpty;
end;
TGenericRecord<T> = class(TInterfacedObject, IKeywordMapping<T>)
@@ -270,6 +272,8 @@ class constructor TKeywordMappingRegistry<T>.Create;
begin
FRegistry := TDictionary<TArray<Integer>, IKeywordMapping<T>>.Create(TEqualityComparer<TArray<Integer>>.Default);
FLock := TSpinLock.Create(False);
FEmpty := Intern([]);
end;
class destructor TKeywordMappingRegistry<T>.Destroy;
Src/Data/Myc.Data.Scalar.pas
+10
View File
@@ -92,6 +92,7 @@ type
class operator Implicit(const A: TScalar): Int64; overload; inline;
class operator Implicit(const A: TScalar): Boolean; overload; inline;
class operator Implicit(const A: TScalar): TDateTime; overload; inline;
class operator Implicit(const A: TScalar): IKeyword; overload; inline;
class function StringToKind(const AName: string): TKind; static;
function ToString: String;
@@ -750,6 +751,15 @@ begin
raise EArgumentException.Create('Operator "xor" requires Ordinal or Boolean arguments.');
end;
class operator TScalar.Implicit(const A: TScalar): IKeyword;
begin
case A.Kind of
TKind.Keyword: Result := TKeywordRegistry.GetKeyword(A.Value.AsInt64);
else
raise EInvalidCast.CreateFmt('Cannot implicitly convert %s to Keyword', [A.Kind.ToString]);
end;
end;
class operator TScalar.LeftShift(const A, B: TScalar): TScalar;
begin
if (A.Kind <> TKind.Ordinal) or (B.Kind <> TKind.Ordinal) then
Src/Data/Myc.Data.Stream.pas
+164 -100
View File
@@ -14,128 +14,143 @@ uses
type
TSignalKind = (skData, skHeartbeat);
// TStreamSignal represents the data packet traveling through the graph.
// It carries the CycleID for synchronization and the actual data payload.
TStreamSignal = record
private
strict private
FCycleID: Int64;
FKind: TSignalKind;
FData: TArray<TScalar.TValue>;
public
constructor Create(AKind: TSignalKind; ACycleID: Int64);
constructor Create(AKind: TSignalKind; ACycleID: Int64; const AData: TArray<TScalar.TValue> = nil);
function ToString: string;
property CycleID: Int64 read FCycleID;
property Kind: TSignalKind read FKind;
property Data: TArray<TScalar.TValue> read FData;
end;
IStreamObserver = interface
procedure OnSignal(const Signal: TStreamSignal);
end;
TSignalProc = reference to procedure(const Signal: TStreamSignal);
TSubscriptionTag = Pointer;
// IStream is now stateless. It defines WHAT is produced (Def), but not
// the history. History must be captured by an observer (Accumulator).
IStream = interface
{$region 'private'}
function GetSeries: IScalarRecordSeries;
function GetDef: IScalarRecordDefinition;
{$endregion}
function Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
function Subscribe(const Observer: TSignalProc): TSubscriptionTag;
procedure Unsubscribe(Tag: TSubscriptionTag);
// Returns the Series of the records this stream produces. Streams are scalar record series. Always!
property Series: IScalarRecordSeries read GetSeries;
{ Returns the definition (schema) of the records this stream produces. }
property Def: IScalarRecordDefinition read GetDef;
end;
// =========================================================================
// BASE STREAM
// Implements storage (Series) and broadcasting (Observers).
// Does not define WHEN data is emitted (Policy).
// =========================================================================
// Implements broadcasting logic. No internal storage of past values.
TCustomDataStream = class(TInterfacedObject, IStream)
strict private
FSeries: IWriteableScalarRecordSeries;
FObservers: TMycNotifyList<IStreamObserver>;
function GetSeries: IScalarRecordSeries;
FDef: IScalarRecordDefinition;
FObservers: TMycNotifyList<TSignalProc>;
function GetDef: IScalarRecordDefinition;
protected
// Derived classes call this to write data and notify listeners.
// ACycleID: The cycle this data belongs to.
{ Broadcasts data to all subscribers. Stateless. }
procedure Emit(const Value: array of TScalar.TValue; ACycleID: Int64);
public
constructor Create(const ADef: IScalarRecordDefinition);
destructor Destroy; override;
// IStream
function Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
function Subscribe(const Observer: TSignalProc): TSubscriptionTag;
procedure Unsubscribe(Tag: TSubscriptionTag);
property Series: IScalarRecordSeries read GetSeries;
property Def: IScalarRecordDefinition read GetDef;
end;
// =========================================================================
// ROOT STREAM (SOURCE)
// Acts as the clock source. Generates new CycleIDs.
// =========================================================================
// The entry point for external data. Generates the global CycleID (Clock).
TRootStream = class(TCustomDataStream)
private
strict private
FLastCycleID: Int64;
public
constructor Create(const ADef: IScalarRecordDefinition);
// Public API to inject data from Delphi
procedure Push(const RowData: TArray<TScalar.TValue>);
{ Injects a new record into the stream system. }
procedure Push(const RowData: array of TScalar.TValue);
end;
// =========================================================================
// PIPE STREAM (NODE)
// Reacts to upstream signals using barrier synchronization.
// =========================================================================
TPipeStream = class; // Forward
// Synchronizes multiple inputs via barrier and executes a transformation.
TPipeStream = class;
TPipeConfig = TArray<TArray<TScalarRecordField>>;
TPipeSource = class(TContainedObject, IStreamObserver)
private
{ TPipeSource acts as a selective accumulator for a single field of a stream. }
TPipeSource = class(TInterfacedObject, ISeries)
public
type
TSignalProc = reference to procedure(const Signal: TStreamSignal);
strict private
FSource: IStream;
FTag: TSubscriptionTag;
FOnSignal: TSignalProc;
FFieldIndex: Integer;
FData: IWriteableSeries;
FLastSeenCycle: Int64;
procedure OnSignal(const Signal: TStreamSignal);
FLookback: Int64;
{ ISeries implementation }
function GetCount: Int64;
function GetItems(Idx: Integer): TScalar;
function GetTotalCount: Int64;
private
procedure HandleSignal(const Signal: TStreamSignal);
protected
procedure Detach;
public
constructor Create(AOwner: TPipeStream; ASource: IStream);
constructor Create(const ASource: IStream; const AField: IKeyword; ALookback: Int64; const AOnSignal: TSignalProc);
destructor Destroy; override;
function IsReady(RequiredLookback: Int64): Boolean;
property LastSeenCycle: Int64 read FLastSeenCycle;
end;
TPipeConfig = TArray<TArray<TScalarRecordField>>;
TPipeStream = class(TCustomDataStream)
public
type
TPipeLambda = reference to function(const Sources: array of ISeries; out Results: array of TScalar.TValue): Boolean;
private
FSources: TArray<TPipeSource>;
FSourceSeries: TArray<ISeries>;
strict private
FSources: TArray<ISeries>;
FResults: TArray<TScalar.TValue>;
FLastFiredCycleID: Int64;
FLookback: Int64;
FLambda: TPipeLambda;
private
procedure CheckBarrierAndFire(CurrentCycle: Int64);
public
constructor Create(
const AConfig: TPipeConfig;
const ADef: IScalarRecordDefinition;
const ASources: TArray<IStream>;
const ALambda: TPipeLambda
const ALambda: TPipeLambda;
ALookback: Int64
);
destructor Destroy; override;
property Lookback: Int64 read FLookback;
end;
implementation
uses
Winapi.Windows;
{ TStreamSignal }
constructor TStreamSignal.Create(AKind: TSignalKind; ACycleID: Int64);
constructor TStreamSignal.Create(AKind: TSignalKind; ACycleID: Int64; const AData: TArray<TScalar.TValue>);
begin
FKind := AKind;
FCycleID := ACycleID;
FData := AData;
end;
function TStreamSignal.ToString: string;
begin
if Kind = skData then
Result := Format('Signal(Data, #%d)', [CycleID])
Result := Format('Signal(Data, #%d, Fields: %d)', [CycleID, Length(Data)])
else
Result := Format('Signal(Heartbeat, #%d)', [CycleID]);
end;
@@ -145,7 +160,7 @@ end;
constructor TCustomDataStream.Create(const ADef: IScalarRecordDefinition);
begin
inherited Create;
FSeries := TScalarRecordSeries.Create(ADef);
FDef := ADef;
end;
destructor TCustomDataStream.Destroy;
@@ -154,12 +169,12 @@ begin
inherited;
end;
function TCustomDataStream.GetSeries: IScalarRecordSeries;
function TCustomDataStream.GetDef: IScalarRecordDefinition;
begin
Result := FSeries;
Result := FDef;
end;
function TCustomDataStream.Subscribe(const Observer: IStreamObserver): TSubscriptionTag;
function TCustomDataStream.Subscribe(const Observer: TSignalProc): TSubscriptionTag;
begin
FObservers.Lock;
try
@@ -171,6 +186,9 @@ end;
procedure TCustomDataStream.Unsubscribe(Tag: TSubscriptionTag);
begin
if Tag = nil then
exit;
FObservers.Lock;
try
FObservers.Unadvise(Tag);
@@ -181,20 +199,23 @@ end;
procedure TCustomDataStream.Emit(const Value: array of TScalar.TValue; ACycleID: Int64);
var
signal: TStreamSignal;
LSignal: TStreamSignal;
LData: TArray<TScalar.TValue>;
I: Integer;
begin
// Convert open array to dynamic array for signal payload
SetLength(LData, Length(Value));
for I := 0 to High(Value) do
LData[I] := Value[I];
LSignal := TStreamSignal.Create(skData, ACycleID, LData);
FObservers.Lock;
try
// 1. Write Data
FSeries.Add(Value);
// 2. Broadcast Signal
signal := TStreamSignal.Create(skData, ACycleID);
FObservers.Notify(
function(const Obs: IStreamObserver): Boolean
function(const Obs: TSignalProc): Boolean
begin
Obs.OnSignal(signal);
Obs(LSignal);
Result := True;
end
);
@@ -211,116 +232,159 @@ begin
FLastCycleID := 0;
end;
procedure TRootStream.Push(const RowData: TArray<TScalar.TValue>);
procedure TRootStream.Push(const RowData: array of TScalar.TValue);
begin
// Root streams increment the global clock
if Length(RowData) <> Def.Count then
raise EArgumentException.Create('Push: Data field count mismatch.');
Inc(FLastCycleID);
Emit(RowData, FLastCycleID);
end;
{ TPipeSource }
constructor TPipeSource.Create(AOwner: TPipeStream; ASource: IStream);
constructor TPipeSource.Create(const ASource: IStream; const AField: IKeyword; ALookback: Int64; const AOnSignal: TSignalProc);
begin
inherited Create(AOwner);
inherited Create;
FSource := ASource;
FOnSignal := AOnSignal;
FLookback := ALookback;
FLastSeenCycle := -1;
FTag := FSource.Subscribe(Self);
FFieldIndex := ASource.Def.IndexOf(AField);
if (FFieldIndex < 0) then
raise EArgumentException.CreateFmt('Field %s not found in source stream.', [AField.Name]);
FData := TScalarSeries.Create(ASource.Def.Items[FFieldIndex]);
FTag := FSource.Subscribe(procedure(const Signal: TStreamSignal) begin HandleSignal(Signal); end);
end;
destructor TPipeSource.Destroy;
begin
FSource.Unsubscribe(FTag);
Detach;
inherited;
end;
procedure TPipeSource.OnSignal(const Signal: TStreamSignal);
procedure TPipeSource.Detach;
begin
if Signal.Kind = skData then
FSource.Unsubscribe(FTag);
FTag := nil;
FOnSignal := nil;
end;
procedure TPipeSource.HandleSignal(const Signal: TStreamSignal);
begin
if (Signal.Kind = skData) then
begin
FLastSeenCycle := Signal.CycleID;
(Controller as TPipeStream).CheckBarrierAndFire(FLastSeenCycle);
FData.Add(Signal.Data[FFieldIndex], FLookback);
if Assigned(FOnSignal) then
FOnSignal(Signal);
end;
end;
function TPipeSource.IsReady(RequiredLookback: Int64): Boolean;
begin
Result := (FLastSeenCycle >= 0) and (FData.Count >= RequiredLookback);
end;
function TPipeSource.GetCount: Int64;
begin
Result := FData.Count;
end;
function TPipeSource.GetItems(Idx: Integer): TScalar;
begin
Result := FData.Items[Idx];
end;
function TPipeSource.GetTotalCount: Int64;
begin
Result := FData.TotalCount;
end;
{ TPipeStream }
constructor TPipeStream.Create(
const AConfig: TPipeConfig;
const ADef: IScalarRecordDefinition;
const ASources: TArray<IStream>;
const ALambda: TPipeLambda
const ALambda: TPipeLambda;
ALookback: Int64
);
var
i, j, n: Integer;
begin
// Pass Definition to base class
inherited Create(ADef);
FLambda := ALambda;
FLookback := ALookback;
FLastFiredCycleID := -1;
SetLength(FSources, Length(ASources));
// Flatten Sources
// Flatten sources from config
n := 0;
for i := 0 to High(AConfig) do
inc(n, Length(AConfig[i]));
SetLength(FSourceSeries, n);
SetLength(FSources, n);
SetLength(FResults, n);
n := 0;
for i := 0 to High(ASources) do
begin
FSources[i] := TPipeSource.Create(Self, ASources[i]);
for j := 0 to High(AConfig[i]) do
begin
// Extract the specific column series
var m := ASources[i].Series.IndexOf(AConfig[i][j].Key);
if m >= 0 then
begin
FSourceSeries[n] := ASources[i].Series[m];
inc(n);
end;
// Each input is wrapped in an accumulator source
FSources[n] :=
TPipeSource.Create(
ASources[i],
AConfig[i][j].Key,
FLookback,
procedure(const Signal: TStreamSignal) begin CheckBarrierAndFire(Signal.CycleID); end
);
inc(n);
end;
end;
end;
destructor TPipeStream.Destroy;
begin
for var i := High(FSources) downto 0 do
FSources[i].Free;
FSources := nil;
FSourceSeries := nil;
for var i := 0 to High(FSources) do
(FSources[i] as TPipeSource).Detach;
inherited;
end;
procedure TPipeStream.CheckBarrierAndFire(CurrentCycle: Int64);
var
resultVal: TArray<TScalar.TValue>;
begin
// We assume thread safety is handled by the caller
const requiredLookback = 10;
// 1. Check if all sources reached this cycle AND have enough history (Warm-up)
for var src in FSources do
if src.LastSeenCycle < CurrentCycle then
exit; // Barrier closed
begin
if src.Count < requiredLookback then
exit;
var LSource := src as TPipeSource;
if LSource.LastSeenCycle < CurrentCycle then
exit;
end;
if FLastFiredCycleID >= CurrentCycle then
exit; // Already fired
Exit;
FLastFiredCycleID := CurrentCycle;
if Assigned(FLambda) then
begin
SetLength(resultVal, Series.Def.Count);
if FLambda(FSourceSeries, resultVal) then
// Execute transformation on synchronized window
if FLambda(FSources, FResults) then
begin
// Pass through CycleID from upstream
Emit(resultVal, FLastFiredCycleID);
Emit(FResults, FLastFiredCycleID);
end;
end;
end;
initialization
TMycNotifyList<IStreamObserver>.ReverseOnNotify := False;
TMycNotifyList<TSignalProc>.ReverseOnNotify := False;
end.