Brummel/MycLib
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Keyword mapping refactoring

Browse Source
This commit is contained in:
Michael Schimmel
2025-12-09 12:59:36 +01:00
parent 43afbd6050
commit f8dc5b945c
13 changed files with 257 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ASTPlayground/ASTPlayground.delphilsp.json
+1 -1
View File
File diff suppressed because one or more lines are too long
AuraTrader/MainForm.pas
+7 -3
View File
@@ -55,6 +55,7 @@ uses
Myc.FMX.Chart,
Myc.Trade.Indicators,
Myc.Trade.Indicators.Common,
Myc.Core.FileCache,
Myc.Trade.DataFeed,
StrategyTest,
TestMethodCallFromRecordParams;
@@ -114,6 +115,7 @@ type
FProcessDone: TState;
FApplication: IAuraApplication;
FModulesItem: TTreeViewItem;
FFileCache: IDataFileCache<TArray<TScalarRecordFeed.TBatch>>;
FFeed: IScalarRecordFeed;
function SelectedSymbol: String;
procedure ExecuteStrategy(const Symbol: String; Timeframe: TTimeframe; const Consumer: IConsumer<TDataPoint<TOhlcItem>>);
@@ -195,6 +197,8 @@ procedure TForm1.FormCreate(Sender: TObject);
begin
FApplication := TMycAuraApplication.Create;
FFileCache := TDataFileCache<TArray<TScalarRecordFeed.TBatch>>.Create;
FModulesItem := TTreeViewItem.Create(Self);
FModulesItem.Text := 'Modules';
@@ -382,15 +386,15 @@ begin
var terminated := TFlag.CreateObserver(FTerminate.Signal).State;
var path := '\\COFFEE\TickData\Pepperstone'; // Or FServer.Path if compatible
FFeed := TScalarRecordFeed.Create(path, '.m1', true, def);
FFeed := TScalarRecordFeed.Create(FFileCache, path, '.m1', true, def);
// Converter: Transforms raw IRecordSeries (Columns) into TArray<TDataPoint>
var ticker :=
FFeed.Ticker.Chain<TDataPoint<Double>>(
function(const Tick: IScalarRecord): TDataPoint<Double>
begin
Result.Time := Tick.ItemByIndex(iTime).Value.AsDouble;
Result.Data := Tick.ItemByIndex(iClose).Value.AsDouble;
Result.Time := Tick[iTime].Value.Value.AsDouble;
Result.Data := Tick[iClose].Value.Value.AsDouble;
end
);
IntfExtract/DataApiToClipboard.bat
+1
View File
@@ -0,0 +1 @@
T:\Myc\IntfExtract\Win64\Debug\ExtractPascalInterfaces.exe -c -dirs T:\Myc\dirs.txt -fc T:\Myc\Src\Myc.Data.*
Src/AST/Myc.Ast.Compiler.TypeChecker.pas
+2 -2
View File
@@ -653,7 +653,7 @@ begin
end
else
begin
var fieldType := TTypes.FromScalarKind(baseType.Definition.Fields[fieldIndex].Value);
var fieldType := TTypes.FromScalarKind(baseType.Definition[fieldIndex].Value);
if baseType.Kind = TStaticTypeKind.stRecord then
elemType := fieldType
else
@@ -670,7 +670,7 @@ begin
FLog.AddError(Format('Member "%s" not found in type %s', [Node.Member.Value.Name, baseType.ToString]), Node);
end
else
elemType := genDef.Fields[fieldIndex].Value;
elemType := genDef[fieldIndex].Value;
end
else
begin
Src/AST/Myc.Ast.Evaluator.pas
+6 -6
View File
@@ -9,7 +9,8 @@ uses
Myc.Data.Scalar,
Myc.Data.Value,
Myc.Ast.Nodes, // Provides EAstException
Myc.Ast.Scope;
Myc.Ast.Scope,
Myc.Data.Keyword;
type
// Exception for runtime errors during script evaluation
@@ -74,7 +75,6 @@ uses
System.TypInfo,
System.Generics.Defaults,
Myc.Ast,
Myc.Data.Keyword,
Myc.Data.Decimal,
Myc.Data.Series,
Myc.Data.Scalar.JSON,
@@ -381,7 +381,7 @@ begin
if def.StartsWith('[') then
begin
var recordDef := TRttiAstHelper.JsonToRecordDefinition(def);
if Length(recordDef.Fields) = 0 then
if recordDef.Count = 0 then
raise EEvaluatorException.Create('Failed to parse record definition from JSON array.');
var recordSeries := TScalarRecordSeries.Create(recordDef);
Result := TDataValue.FromRecordSeries(recordSeries);
@@ -440,12 +440,12 @@ begin
raise EEvaluatorException
.CreateFmt('Index %d is out of bounds for series with %d elements.', [index, recSeries.TotalCount]);
fieldCount := Length(recSeries.Def.Fields);
fieldCount := recSeries.Def.Count;
SetLength(values, fieldCount);
for i := 0 to fieldCount - 1 do
begin
key := recSeries.Def.Fields[i].Key;
key := recSeries.Def.Items[i].Key;
memberSeries := recSeries[key];
scalarValue := memberSeries[index];
values[i] := scalarValue.Value;
@@ -477,7 +477,7 @@ begin
if fieldIndex < 0 then
raise EEvaluatorException.CreateFmt('Member ":%s" not found in record.', [Node.Member.Value.Name]);
Result := rec.Fields[fieldIndex].Value;
Result := rec.Items[fieldIndex].Value;
end;
else
raise EEvaluatorException.Create('Member access operator `.` is not supported for this value type.');
Src/AST/Myc.Ast.Types.pas
+22 -15
View File
@@ -504,12 +504,12 @@ begin
if (not Assigned(Self.FDefinition)) or (not Assigned(otherDef)) then
exit(False);
if Length(Self.FDefinition.Fields) <> Length(otherDef.Fields) then
if Self.FDefinition.Count <> otherDef.Count then
exit(False);
for i := 0 to High(Self.FDefinition.Fields) do
for i := 0 to Self.FDefinition.Count - 1 do
begin
if (Self.FDefinition.Fields[i].Key <> otherDef.Fields[i].Key) or (Self.FDefinition.Fields[i].Value <> otherDef.Fields[i].Value) then
if (Self.FDefinition[i].Key <> otherDef[i].Key) or (Self.FDefinition[i].Value <> otherDef[i].Value) then
exit(False);
end;
@@ -518,13 +518,15 @@ end;
function TRecordType.GetHashCode: Integer;
var
i: Integer;
field: TPair<IKeyword, TScalar.TKind>;
begin
Result := Ord(GetKind);
if Assigned(FDefinition) then
begin
for field in FDefinition.Fields do
for i := 0 to FDefinition.Count - 1 do
begin
field := FDefinition[i];
var hash := TEqualityComparer<IKeyword>.Default.GetHashCode(field.Key);
Result := THashBobJenkins.GetHashValue(hash, SizeOf(Pointer), Result);
var data := Ord(field.Value);
@@ -536,14 +538,16 @@ end;
function TRecordType.ToString: string;
var
i: Integer;
field: TPair<IKeyword, TScalar.TKind>;
begin
Result := GetKind.ToString + '{';
if Assigned(FDefinition) then
begin
for i := 0 to High(FDefinition.Fields) do
for i := 0 to FDefinition.Count - 1 do
begin
Result := Result + FDefinition.Fields[i].Key.Name + ': ' + FDefinition.Fields[i].Value.ToString;
if i < High(FDefinition.Fields) then
field := FDefinition[i];
Result := Result + field.Key.Name + ': ' + field.Value.ToString;
if i < FDefinition.Count - 1 then
Result := Result + ', ';
end;
end;
@@ -591,13 +595,12 @@ begin
if (not Assigned(Self.FDefinition)) or (not Assigned(otherDef)) then
exit(False);
if Length(Self.FDefinition.Fields) <> Length(otherDef.Fields) then
if Self.FDefinition.Count <> otherDef.Count then
exit(False);
for i := 0 to High(Self.FDefinition.Fields) do
for i := 0 to Self.FDefinition.Count - 1 do
begin
if (Self.FDefinition.Fields[i].Key <> otherDef.Fields[i].Key)
or (not Self.FDefinition.Fields[i].Value.IsEqual(otherDef.Fields[i].Value)) then
if (Self.FDefinition[i].Key <> otherDef[i].Key) or (not Self.FDefinition[i].Value.IsEqual(otherDef[i].Value)) then
exit(False);
end;
@@ -606,13 +609,15 @@ end;
function TGenericRecordType.GetHashCode: Integer;
var
i: Integer;
field: TPair<IKeyword, IStaticType>;
begin
Result := Ord(stGenericRecord);
if Assigned(FDefinition) then
begin
for field in FDefinition.Fields do
for i := 0 to FDefinition.Count - 1 do
begin
field := FDefinition[i];
var data := TEqualityComparer<IKeyword>.Default.GetHashCode(field.Key);
Result := THashBobJenkins.GetHashValue(data, SizeOf(Pointer), Result);
if Assigned(field.Value) then
@@ -627,14 +632,16 @@ end;
function TGenericRecordType.ToString: string;
var
i: Integer;
field: TPair<IKeyword, IStaticType>;
begin
Result := GetKind.ToString + '{';
if Assigned(FDefinition) then
begin
for i := 0 to High(FDefinition.Fields) do
for i := 0 to FDefinition.Count - 1 do
begin
Result := Result + FDefinition.Fields[i].Key.Name + ': ' + FDefinition.Fields[i].Value.ToString;
if i < High(FDefinition.Fields) then
field := FDefinition[i];
Result := Result + field.Key.Name + ': ' + field.Value.ToString;
if i < FDefinition.Count - 1 then
Result := Result + ', ';
end;
end;
Src/Data/Myc.Data.Keyword.pas
+97 -74
View File
@@ -4,15 +4,17 @@ interface
uses
System.SysUtils,
System.Classes,
System.Generics.Collections,
System.Generics.Defaults,
System.SyncObjs;
type
// The runtime representation of an interned keyword
IKeyword = interface
{$region 'private'}
function GetIdx: Integer;
function GetName: string;
{$endregion}
property Idx: Integer read GetIdx;
property Name: string read GetName;
end;
@@ -33,10 +35,8 @@ type
strict private
class var
FLock: TSpinLock;
class var
FRegistry: TDictionary<string, Integer>;
class var
FReverseMap: TList<IKeyword>; // Use TList
FReverseMap: TList<IKeyword>;
class constructor Create;
class destructor Destroy;
public
@@ -49,12 +49,18 @@ type
// Defines a mapping from Keywords to a generic value T
IKeywordMapping<T> = interface
{$region 'private'}
function GetFields: TArray<TPair<IKeyword, T>>;
function GetItems(Idx: Integer): TPair<IKeyword, T>;
function GetFields(const Key: IKeyword): T;
function GetCount: Integer;
{$endregion}
// Finds the 0-based index for a given keyword. Returns -1 if not found.
function IndexOf(const Key: IKeyword): Integer;
// Gets all fields in the mapping.
property Fields: TArray<TPair<IKeyword, T>> read GetFields;
// Access value by Keyword (raises exception if not found)
property Fields[const Key: IKeyword]: T read GetFields;
// Access Key-Value pair by Index (0..Count-1)
property Items[Idx: Integer]: TPair<IKeyword, T> read GetItems; default;
property Count: Integer read GetCount;
end;
// Factory for creating IKeywordMapping instances
@@ -67,7 +73,11 @@ type
FMap: TArray<Integer>;
FFields: TArray<TPair<IKeyword, T>>;
FFirst, FLast: Integer;
function GetFields: TArray<TPair<IKeyword, T>>;
function GetItem(Idx: Integer): TPair<IKeyword, T>;
function GetItems(Idx: Integer): TPair<IKeyword, T>;
function GetFields(const Key: IKeyword): T;
function GetCount: Integer;
public
// Expects AFields to be in the desired (canonical) order
constructor Create(const AFields: TArray<TPair<IKeyword, T>>);
@@ -77,7 +87,6 @@ type
// Cache implementation
class var
FLock: TSpinLock;
class var
FRegistry: TDictionary<TArray<Integer>, IKeywordMapping<T>>;
class constructor Create;
class destructor Destroy;
@@ -89,10 +98,6 @@ type
implementation
uses
WinApi.Windows,
System.Generics.Defaults; // For TComparer<Integer>
{ TKeywordRegistry.TKeyword }
constructor TKeywordRegistry.TKeyword.Create(const AName: string; AIdx: Integer);
@@ -116,9 +121,9 @@ end;
class constructor TKeywordRegistry.Create;
begin
FLock := TSpinLock.Create(false);
FRegistry := TDictionary<string, Integer>.Create;
FReverseMap := TList<IKeyword>.Create;
FLock := TSpinLock.Create(False);
end;
class destructor TKeywordRegistry.Destroy;
@@ -127,18 +132,6 @@ begin
FReverseMap.Free;
end;
class function TKeywordRegistry.GetName(AIdx: Integer): string;
begin
FLock.Enter;
try
// Check bounds
if (AIdx >= 0) and (AIdx < FReverseMap.Count) then
Result := FReverseMap[AIdx].Name;
finally
FLock.Exit;
end;
end;
class function TKeywordRegistry.Intern(const AName: string): IKeyword;
var
idx: Integer;
@@ -147,72 +140,118 @@ begin
try
if not FRegistry.TryGetValue(AName, idx) then
begin
Result := TKeyword.Create(AName, FRegistry.Count);
FRegistry.Add(AName, FReverseMap.Add(Result));
idx := FReverseMap.Count;
Result := TKeyword.Create(AName, idx);
FReverseMap.Add(Result);
FRegistry.Add(AName, idx);
end
else
begin
Result := FReverseMap[idx];
end;
finally
FLock.Exit;
end;
end;
{ TKeywordMappingRegistry<T>.TKeywordMapping<T> }
class function TKeywordRegistry.GetName(AIdx: Integer): string;
begin
if (AIdx >= 0) and (AIdx < FReverseMap.Count) then
Result := FReverseMap[AIdx].Name
else
Result := '';
end;
{ TKeywordMappingRegistry<T>.TKeywordMapping }
constructor TKeywordMappingRegistry<T>.TKeywordMapping.Create(const AFields: TArray<TPair<IKeyword, T>>);
var
i, idx: Integer;
i: Integer;
keyIdx: Integer;
begin
inherited Create;
FFields := AFields;
if Length(FFields) = 0 then
begin
FFirst := 0;
FLast := -1;
if Length(FFields) = 0 then
exit;
FMap := nil;
Exit;
end;
// Find min/max Idx for the map. Fields are *not* assumed to be sorted.
FFirst := FFields[0].Key.Idx;
FLast := FFirst;
for i := 1 to High(FFields) do
begin
idx := FFields[i].Key.Idx;
if FFirst > idx then
FFirst := idx;
if FLast < idx then
FLast := idx;
keyIdx := FFields[i].Key.Idx;
if keyIdx < FFirst then
FFirst := keyIdx;
if keyIdx > FLast then
FLast := keyIdx;
end;
SetLength(FMap, FLast - FFirst + 1);
for i := 0 to High(FMap) do
FMap[i] := -1;
// Populate map based on the *original field order*
for i := 0 to High(FFields) do
FMap[FFields[i].Key.Idx - FFirst] := i;
begin
keyIdx := FFields[i].Key.Idx;
FMap[keyIdx - FFirst] := i;
end;
end;
function TKeywordMappingRegistry<T>.TKeywordMapping.GetFields: TArray<TPair<IKeyword, T>>;
function TKeywordMappingRegistry<T>.TKeywordMapping.GetCount: Integer;
begin
Result := FFields;
Result := Length(FFields);
end;
// Internal helper required for interface implementation match if defined in stricter mode
function TKeywordMappingRegistry<T>.TKeywordMapping.GetItem(Idx: Integer): TPair<IKeyword, T>;
begin
Result := FFields[Idx];
end;
// Interface method for Items property
function TKeywordMappingRegistry<T>.TKeywordMapping.GetItems(Idx: Integer): TPair<IKeyword, T>;
begin
Result := FFields[Idx];
end;
// Interface method for Fields property
function TKeywordMappingRegistry<T>.TKeywordMapping.GetFields(const Key: IKeyword): T;
var
idx: Integer;
begin
idx := IndexOf(Key);
if idx < 0 then
raise EArgumentException.CreateFmt('Keyword "%s" not found in mapping.', [Key.Name]);
Result := FFields[idx].Value;
end;
function TKeywordMappingRegistry<T>.TKeywordMapping.IndexOf(const Key: IKeyword): Integer;
var
idx: Integer;
keyIdx, mapIdx: Integer;
begin
idx := Key.Idx - FFirst;
if (idx < 0) or (Idx > High(FMap)) then
exit(-1);
keyIdx := Key.Idx;
if (keyIdx < FFirst) or (keyIdx > FLast) then
Exit(-1);
Result := FMap[idx];
mapIdx := keyIdx - FFirst;
if (mapIdx < Length(FMap)) then
Result := FMap[mapIdx]
else
Result := -1;
end;
{ TKeywordMappingRegistry<T> }
class constructor TKeywordMappingRegistry<T>.Create;
begin
FLock := TSpinLock.Create(false);
FRegistry := TDictionary<TArray<Integer>, IKeywordMapping<T>>.Create;
FRegistry := TDictionary<TArray<Integer>, IKeywordMapping<T>>.Create(TEqualityComparer<TArray<Integer>>.Default);
FLock := TSpinLock.Create(False);
end;
class destructor TKeywordMappingRegistry<T>.Destroy;
@@ -222,36 +261,20 @@ end;
class function TKeywordMappingRegistry<T>.Intern(const AFields: TArray<TPair<IKeyword, T>>): IKeywordMapping<T>;
var
key: TArray<Integer>;
newMapping: IKeywordMapping<T>;
signature: TArray<Integer>;
i: Integer;
begin
SetLength(key, Length(AFields));
for var i := 0 to High(key) do
key[i] := AFields[i].Key.Idx;
SetLength(signature, Length(AFields));
for i := 0 to High(AFields) do
signature[i] := AFields[i].Key.Idx;
// 2. Lock
FLock.Enter;
try
// 3. Check cache
if FRegistry.TryGetValue(key, Result) then
exit; // Found it, exit
finally
FLock.Exit;
if not FRegistry.TryGetValue(signature, Result) then
begin
Result := TKeywordMapping.Create(AFields);
FRegistry.Add(signature, Result);
end;
// 4. Not in cache. Create (O(N+M)) - OUTSIDE lock
newMapping := TKeywordMapping.Create(AFields) as IKeywordMapping<T>;
// 5. Lock again to add
FLock.Enter;
try
// 6. Check again (double-check)
if FRegistry.TryGetValue(key, Result) then
exit; // Another thread won the race
// 7. Add our new mapping.
Result := newMapping;
FRegistry.Add(key, Result);
finally
FLock.Exit;
end;
Src/Data/Myc.Data.Records.pas
+30 -5
View File
@@ -3,6 +3,7 @@ unit Myc.Data.Records;
interface
uses
System.SysUtils,
System.Generics.Collections,
Myc.Data.Keyword,
Myc.Data.Value;
@@ -12,7 +13,11 @@ type
TDynamicRecord = class(TInterfacedObject, IKeywordMapping<TDataValue>)
private
FFields: TArray<TPair<IKeyword, TDataValue>>;
function GetFields: TArray<TPair<IKeyword, TDataValue>>;
// IKeywordMapping implementation
function GetItems(Idx: Integer): TPair<IKeyword, TDataValue>;
function GetFields(const Key: IKeyword): TDataValue;
function GetCount: Integer;
public
constructor Create(const AFields: TArray<TPair<IKeyword, TDataValue>>);
function IndexOf(const Key: IKeyword): Integer;
@@ -28,20 +33,40 @@ begin
FFields := AFields;
end;
function TDynamicRecord.GetFields: TArray<TPair<IKeyword, TDataValue>>;
function TDynamicRecord.GetCount: Integer;
begin
Result := FFields;
Result := Length(FFields);
end;
function TDynamicRecord.GetFields(const Key: IKeyword): TDataValue;
var
idx: Integer;
begin
idx := IndexOf(Key);
if idx < 0 then
raise EArgumentException.CreateFmt('Field "%s" not found in dynamic record.', [Key.Name]);
Result := FFields[idx].Value;
end;
function TDynamicRecord.GetItems(Idx: Integer): TPair<IKeyword, TDataValue>;
begin
Result := FFields[Idx];
end;
function TDynamicRecord.IndexOf(const Key: IKeyword): Integer;
var
i: Integer;
begin
// Linear search (O(n)). Fast enough for typical records.
// Keyword comparison is integer-based (Idx), so this is efficient.
for Result := 0 to High(FFields) do
for i := 0 to High(FFields) do
begin
if FFields[Result].Key.Idx = Key.Idx then
if FFields[i].Key.Idx = Key.Idx then
begin
Result := i;
exit;
end;
end;
Result := -1;
end;
Src/Data/Myc.Data.Scalar.pas
+10 -9
View File
@@ -143,6 +143,7 @@ type
// A field definition for a scalar record.
TScalarRecordField = TPair<IKeyword, TScalar.TKind>;
IScalarRecordDefinition = IKeywordMapping<TScalar.TKind>;
IScalarRecord = IKeywordMapping<TScalar>;
TScalarRecordRegistry = TKeywordMappingRegistry<TScalar.TKind>;
// A record of scalar values, based on a definition.
@@ -804,7 +805,7 @@ end;
constructor TScalarRecord.Create(const ADef: IScalarRecordDefinition; const AFields: TArray<TScalar.TValue>);
begin
Assert(Length(ADef.Fields) = Length(AFields), 'Field definition and value count must match.');
Assert(ADef.Count = Length(AFields), 'Field definition and value count must match.');
FDef := ADef;
FFields := AFields;
end;
@@ -827,17 +828,17 @@ begin
if idx < 0 then
raise EArgumentException.CreateFmt('Field ":%s" not found in record definition.', [Key.Name]);
Result.Create(FDef.Fields[idx].Value, FFields[idx]);
Result.Create(FDef[idx].Value, FFields[idx]);
end;
{ TScalarRecordSeries }
constructor TScalarRecordSeries.Create(const ADef: IScalarRecordDefinition);
begin
Assert(Length(ADef.Fields) > 0);
Assert(ADef.Count > 0);
FDef := ADef;
FTotalCount := 0;
SetLength(FFields, Length(FDef.Fields));
SetLength(FFields, FDef.Count);
for var i := 0 to High(FFields) do
FFields[i] := TMemberSeries.Create(Self, i);
end;
@@ -851,7 +852,7 @@ end;
procedure TScalarRecordSeries.Add(const Item: TScalarRecord; Lookback: Int64 = -1);
begin
FArray.Add(Item.Fields, Length(FDef.Fields) * Lookback);
FArray.Add(Item.Fields, FDef.Count * Integer(Lookback));
inc(FTotalCount);
end;
@@ -862,7 +863,7 @@ type
var
len: Integer;
begin
len := Length(FDef.Fields);
len := FDef.Count;
// Use Slice to pass the raw data as an open array without copying.
FArray.Add(Slice(PValueArr(@Data)^, len), len * Integer(Lookback));
inc(FTotalCount);
@@ -879,7 +880,7 @@ end;
function TScalarRecordSeries.GetCount: Int64;
begin
Result := FArray.Count div Length(FDef.Fields);
Result := FArray.Count div FDef.Count;
end;
function TScalarRecordSeries.GetDef: IScalarRecordDefinition;
@@ -889,7 +890,7 @@ end;
function TScalarRecordSeries.GetItemRef(Idx: Integer): TChunkArray<TScalar.TValue>.PT;
begin
var len := Length(FDef.Fields);
var len := FDef.Count;
Result := FArray.ItemRef[(FArray.Count - len) - (Idx * len)];
end;
@@ -958,7 +959,7 @@ end;
constructor TScalarRecordSeries.TMemberSeries.Create(ARecordSeries: TScalarRecordSeries; AElementIdx: Integer);
begin
inherited Create(ARecordSeries);
FKind := ARecordSeries.FDef.Fields[AElementIdx].Value;
FKind := ARecordSeries.FDef[AElementIdx].Value;
FOffset := sizeof(TScalar.TValue) * AElementIdx;
end;
Src/Data/Myc.Data.Value.pas
+7 -3
View File
@@ -300,6 +300,7 @@ var
field: TScalarRecordField;
genField: TPair<IKeyword, TDataValue>;
first: Boolean;
i: Integer;
begin
case FKind of
vkScalar: Result := FScalar.ToString;
@@ -317,10 +318,11 @@ begin
try
sb.Append('{');
first := True;
for field in series.Def.Fields do
for i := 0 to series.Def.Count - 1 do
begin
if not first then
sb.Append(',');
field := series.Def[i];
sb.Append(field.Key.Name);
first := False;
end;
@@ -338,10 +340,11 @@ begin
try
sb.Append('{');
first := True;
for field in rec.Def.Fields do
for i := 0 to rec.Def.Count - 1 do
begin
if not first then
sb.Append(',');
field := rec.Def[i];
sb.Append(field.Key.Name);
first := False;
end;
@@ -360,10 +363,11 @@ begin
try
sb.Append('{');
first := True;
for genField in rec.Fields do
for i := 0 to rec.Count - 1 do
begin
if not first then
sb.Append(',');
genField := rec[i];
sb.Append(genField.Key.Name);
first := False;
end;
Src/Myc.Core.FileCache.pas
+14 -11
View File
@@ -11,12 +11,17 @@ uses
Myc.Futures;
type
TLoaderFunc<T> = reference to function(const Filename: String): TFuture<T>;
IDataFileCache<T> = interface
procedure Clear;
function GetOrAdd(const AFileName: string; const Loader: TLoaderFunc<T>): TFuture<T>;
end;
// Manages a cache for time-series data files loaded asynchronously.
// The cache is activated by assigning a function to the IsMemoryLow property.
// It evicts least-recently-used items when memory is low.
TDataFileCache<T> = class
type
TLoaderFunc = reference to function(const Filename: String): TFuture<T>;
TDataFileCache<T> = class(TInterfacedObject, IDataFileCache<T>)
private
type
// Holds metadata for a cached file.
@@ -29,10 +34,9 @@ type
end;
private
FCachedFiles: TDictionary<String, TCachedFile>;
FLoader: TLoaderFunc;
FLock: TCriticalSection;
public
constructor Create(const ALoader: TLoaderFunc);
constructor Create;
destructor Destroy; override;
// Removes all items from the cache, waiting for any pending operations to complete.
@@ -40,7 +44,7 @@ type
// Retrieves a data future from the cache or uses the provided loader function to create and cache it.
// Caching is only active if the IsMemoryLow property is assigned.
function GetOrAdd(const AFileName: string): TFuture<T>;
function GetOrAdd(const AFileName: string; const Loader: TLoaderFunc<T>): TFuture<T>;
end;
var
@@ -56,10 +60,9 @@ uses
{ TDataFileCache<T> }
constructor TDataFileCache<T>.Create(const ALoader: TLoaderFunc);
constructor TDataFileCache<T>.Create;
begin
inherited Create;
FLoader := ALoader;
FLock := TCriticalSection.Create;
FCachedFiles := TObjectDictionary<String, TCachedFile>.Create([doOwnsValues]);
end;
@@ -85,7 +88,7 @@ begin
end;
end;
function TDataFileCache<T>.GetOrAdd(const AFileName: string): TFuture<T>;
function TDataFileCache<T>.GetOrAdd(const AFileName: string; const Loader: TLoaderFunc<T>): TFuture<T>;
var
fileList: TArray<TCachedFile>;
i: Integer;
@@ -96,7 +99,7 @@ begin
// Simply execute the loader function and return the result.
if not Assigned(IsMemoryLow) then
begin
Result := FLoader(AFilename);
Result := Loader(AFilename);
exit;
end;
@@ -150,7 +153,7 @@ begin
end;
// Cache miss or stale entry. Execute the loader function.
Result := FLoader(AFilename);
Result := Loader(AFilename);
// Create a new cache entry for the loaded future.
cachedFile := TCachedFile.Create;
Src/Myc.Trade.DataFeed.pas
+47 -47
View File
@@ -20,26 +20,17 @@ uses
type
TImportConverter = reference to procedure(const Src: Pointer; Dst: TScalar.PValue);
// Interface representing a read-only view of a specific data row.
IScalarRecord = interface
function GetDef: IScalarRecordDefinition;
function GetItems(const Key: IKeyword): TScalar;
function ItemByIndex(Idx: Integer): TScalar;
property Def: IScalarRecordDefinition read GetDef;
property Items[const Key: IKeyword]: TScalar read GetItems; default;
end;
IScalarRecordFeed = interface
procedure ClearCache;
{$region 'private'}
function GetTicker: TProducer<IScalarRecord>;
{$endregion}
procedure SetImportOptions(RecordSize: Integer; const Converter: TImportConverter);
function EnumerateSymbols: TFuture<TArray<String>>;
// Streaming processing: Reads data from disk and broadcasts it via Ticker
function ProcessData(const Symbol: String; const Terminated: TState): TState;
function GetTicker: TProducer<IScalarRecord>;
property Ticker: TProducer<IScalarRecord> read GetTicker;
end;
@@ -79,7 +70,7 @@ type
FFileExtension: String;
FIsCompressed: Boolean;
FCachedFiles: TDataFileCache<TArray<TBatch>>;
FCachedFiles: IDataFileCache<TArray<TBatch>>;
FSymbols: TFuture<TDictionary<String, TArray<TDataFile>>>;
// Producer implementation
@@ -116,10 +107,14 @@ type
function GetTicker: TProducer<IScalarRecord>;
public
constructor Create(const APath: String; const AExtension: String; IsCompressed: Boolean; const ADef: IScalarRecordDefinition);
constructor Create(
const ACachedFiles: IDataFileCache<TArray<TBatch>>;
const APath, AExtension: String;
IsCompressed: Boolean;
const ADef: IScalarRecordDefinition
);
destructor Destroy; override;
procedure AfterConstruction; override;
procedure ClearCache;
procedure SetImportOptions(RecordSize: Integer; const Converter: TImportConverter);
function EnumerateSymbols: TFuture<TArray<String>>;
@@ -152,14 +147,15 @@ type
private
FDef: IScalarRecordDefinition;
FData: TScalar.PValue;
// IKeywordMapping<TScalar.TValue> implementation
function IndexOf(const Key: IKeyword): Integer;
function GetCount: Integer;
function GetItems(Idx: Integer): TPair<IKeyword, TScalar>;
function GetFields(const Key: IKeyword): TScalar;
public
constructor Create(const ADef: IScalarRecordDefinition);
procedure SetData(const AData: TScalar.PValue); inline;
function ItemByIndex(Idx: Integer): TScalar; inline;
function GetDef: IScalarRecordDefinition;
function GetItems(const Key: IKeyword): TScalar;
end;
{ TScalarRecordView }
@@ -175,26 +171,34 @@ begin
FData := AData;
end;
function TScalarRecordView.GetDef: IScalarRecordDefinition;
function TScalarRecordView.GetCount: Integer;
begin
Result := FDef;
Result := FDef.Count;
end;
function TScalarRecordView.GetItems(const Key: IKeyword): TScalar;
function TScalarRecordView.GetFields(const Key: IKeyword): TScalar;
var
idx: Integer;
begin
// Optimization: Depending on hot-path usage, one could cache the Fields array
// or use direct index access if the definition allows.
var idx := FDef.IndexOf(Key);
idx := FDef.IndexOf(Key);
if idx < 0 then
exit(TScalar.Create(TScalar.TKind.Ordinal, Default(TScalar.TValue)));
exit(Default(TScalar));
Result := ItemByIndex(idx);
// Return the raw TValue at the offset
Result.Kind := FDef[idx].Value;
Result.Value := TScalar.PValue(NativeUInt(FData) + NativeUInt(idx * SizeOf(TScalar.TValue)))^;
end;
function TScalarRecordView.ItemByIndex(Idx: Integer): TScalar;
function TScalarRecordView.GetItems(Idx: Integer): TPair<IKeyword, TScalar>;
begin
// Calculate pointer offset
Result.Create(FDef.Fields[Idx].Value, TScalar.PValue(NativeUInt(FData) + NativeUInt(idx * SizeOf(TScalar.TValue)))^);
Result.Key := FDef[Idx].Key;
Result.Value.Kind := FDef[idx].Value;
Result.Value.Value := TScalar.PValue(NativeUInt(FData) + NativeUInt(Idx * SizeOf(TScalar.TValue)))^;
end;
function TScalarRecordView.IndexOf(const Key: IKeyword): Integer;
begin
Result := FDef.IndexOf(Key);
end;
{ TScalarRecordFeed.TDataFile }
@@ -227,8 +231,8 @@ end;
{ TScalarRecordFeed }
constructor TScalarRecordFeed.Create(
const APath: String;
const AExtension: String;
const ACachedFiles: IDataFileCache<TArray<TBatch>>;
const APath, AExtension: String;
IsCompressed: Boolean;
const ADef: IScalarRecordDefinition
);
@@ -236,6 +240,7 @@ begin
inherited Create;
FPath := APath;
FDef := ADef;
FCachedFiles := ACachedFiles;
FFileExtension := AExtension;
FIsCompressed := IsCompressed;
FLookback := -1;
@@ -244,15 +249,12 @@ begin
FTicker := TMycContainedProducer<IScalarRecord>.Create(Self);
FCachedFiles :=
TDataFileCache<TArray<TBatch>>
.Create(function(const Filename: String): TFuture<TArray<TBatch>> begin Result := DoLoad(Filename, FLoadGate); end);
FCachedFiles := TDataFileCache<TArray<TBatch>>.Create;
end;
destructor TScalarRecordFeed.Destroy;
begin
ClearCache;
FCachedFiles.Free;
FCachedFiles.Clear;
FTicker.Free;
inherited Destroy;
end;
@@ -263,16 +265,10 @@ begin
UpdateSymbols;
end;
procedure TScalarRecordFeed.ClearCache;
begin
FCachedFiles.Clear;
end;
procedure TScalarRecordFeed.SetImportOptions(RecordSize: Integer; const Converter: TImportConverter);
begin
FImportRecordSize := RecordSize;
FImportConverter := Converter;
ClearCache;
end;
function TScalarRecordFeed.GetPath: String;
@@ -443,7 +439,11 @@ end;
function TScalarRecordFeed.LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TBatch>>;
begin
if DataFile.IsValid then
Result := FCachedFiles.GetOrAdd(DataFile.GetFullFileName)
Result :=
FCachedFiles.GetOrAdd(
DataFile.GetFullFileName,
function(const Filename: String): TFuture<TArray<TBatch>> begin Result := DoLoad(Filename, FLoadGate); end
)
else
Result := TFuture<TArray<TBatch>>.Construct(TArray<TBatch>(nil));
end;
@@ -551,7 +551,7 @@ begin
InputStream.Position := 0;
fileSize := InputStream.Size;
fieldsPerRecord := Length(FDef.Fields);
fieldsPerRecord := FDef.Count;
internalRecordSize := fieldsPerRecord * sizeof(TScalar.TValue);
if (FImportRecordSize > 0) and Assigned(FImportConverter) then
Src/Myc.Trade.DataStream.pas
+10 -9
View File
@@ -180,14 +180,7 @@ begin
inherited Create;
FPath := APath;
FCachedFiles :=
TDataFileCache<TArray<TArray<TDataPoint<T>>>>.Create(
function(const Filename: String): TFuture<TArray<TArray<TDataPoint<T>>>>
begin
// Pass the private load gate to the virtual DoLoad method.
Result := DoLoad(Filename, FLoadGate);
end
);
FCachedFiles := TDataFileCache<TArray<TArray<TDataPoint<T>>>>.Create;
end;
destructor TDataServer<T>.Destroy;
@@ -349,7 +342,15 @@ end;
function TDataServer<T>.LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TArray<TDataPoint<T>>>>;
begin
if DataFile.IsValid then
Result := FCachedFiles.GetOrAdd(DataFile.GetFullFileName)
Result :=
FCachedFiles.GetOrAdd(
DataFile.GetFullFileName,
function(const Filename: String): TFuture<TArray<TArray<TDataPoint<T>>>>
begin
// Pass the private load gate to the virtual DoLoad method.
Result := DoLoad(Filename, FLoadGate);
end
)
else
Result := TFuture<TArray<TArray<TDataPoint<T>>>>.Construct(TArray<TArray<TDataPoint<T>>>(nil));
end;