This commit is contained in:
Michael Schimmel
2025-12-08 10:54:22 +01:00
parent 95de9a155e
commit 7aa0056799
2 changed files with 79 additions and 79 deletions
+1 -1
View File
@@ -3,7 +3,7 @@ unit Myc.Data.Records;
interface interface
uses uses
system.generics.collections, System.Generics.Collections,
Myc.Data.Keyword, Myc.Data.Keyword,
Myc.Data.Value; Myc.Data.Value;
+78 -78
View File
@@ -27,7 +27,7 @@ type
// Aura files // Aura files
// Represents metadata for a single data file. // Represents metadata for a single data file.
TAuraDataFile = record TDataFile = record
private private
FExtension: String; FExtension: String;
FPath: String; FPath: String;
@@ -49,12 +49,12 @@ type
end; end;
// Generic server for loading and managing sequential time-series data from files. // Generic server for loading and managing sequential time-series data from files.
TAuraDataServer<T: record> = class(TInterfacedObject, IDataServer<T>) TDataServer<T: record> = class(TInterfacedObject, IDataServer<T>)
private private
FCachedFiles: TDataFileCache<TArray<TArray<TDataPoint<T>>>>; FCachedFiles: TDataFileCache<TArray<TArray<TDataPoint<T>>>>;
FPath: String; FPath: String;
// The cache now holds a sorted array of all files for each symbol. // The cache now holds a sorted array of all files for each symbol.
FSymbols: TFuture<TDictionary<String, TArray<TAuraDataFile>>>; FSymbols: TFuture<TDictionary<String, TArray<TDataFile>>>;
function GetPath: String; function GetPath: String;
function ProcessChunks( function ProcessChunks(
@@ -64,7 +64,7 @@ type
): TState; ): TState;
function ProcessFile( function ProcessFile(
FileInfo: TAuraDataFile; FileInfo: TDataFile;
const DataFile: TFuture<TArray<TArray<TDataPoint<T>>>>; const DataFile: TFuture<TArray<TArray<TDataPoint<T>>>>;
const Terminated: TState; const Terminated: TState;
Processor: IConsumer<TArray<TDataPoint<T>>> Processor: IConsumer<TArray<TDataPoint<T>>>
@@ -78,7 +78,7 @@ type
// Used by cache to actually load an uncached file. Now virtual and abstract. // Used by cache to actually load an uncached file. Now virtual and abstract.
function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<T>>>>; virtual; abstract; function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<T>>>>; virtual; abstract;
// Gets the next consecutive data file from the cached, sorted list. // Gets the next consecutive data file from the cached, sorted list.
function GetNextFile(const Curr: TAuraDataFile): TAuraDataFile; virtual; function GetNextFile(const Curr: TDataFile): TDataFile; virtual;
public public
constructor Create(const APath: String); constructor Create(const APath: String);
@@ -87,12 +87,12 @@ type
procedure ClearCache; procedure ClearCache;
procedure UpdateSymbols; procedure UpdateSymbols;
// Scans a directory to find the oldest file for a specific symbol. // Scans a directory to find the oldest file for a specific symbol.
function FindFirstFile(const Symbol: string): TFuture<TAuraDataFile>; function FindFirstFile(const Symbol: string): TFuture<TDataFile>;
function ParseFileName(const FileName: string): TAuraDataFile; virtual; abstract; function ParseFileName(const FileName: string): TDataFile; virtual; abstract;
function EnumerateSymbols: TFuture<TArray<String>>; function EnumerateSymbols: TFuture<TArray<String>>;
// Load data file and split content into chunks. // Load data file and split content into chunks.
function LoadDataFile(const DataFile: TAuraDataFile): TFuture<TArray<TArray<TDataPoint<T>>>>; function LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TArray<TDataPoint<T>>>>;
function ProcessData(const Symbol: String; const Terminated: TState; const Processor: IConsumer<TArray<TDataPoint<T>>>): TState; function ProcessData(const Symbol: String; const Terminated: TState; const Processor: IConsumer<TArray<TDataPoint<T>>>): TState;
@@ -101,23 +101,23 @@ type
// Aura tick data file Ask-Bid // Aura tick data file Ask-Bid
TAuraAskBidFileItem = packed record TAskBidFileItem = packed record
Ask: Single; Ask: Single;
Bid: Single; Bid: Single;
end; end;
TAuraTABFileServer = class(TAuraDataServer<TAuraAskBidFileItem>) TAskBidFileServer = class(TDataServer<TAskBidFileItem>)
private private
class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>; class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAskBidFileItem>>>;
class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>; class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAskBidFileItem>>>;
protected protected
function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>; override; function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TAskBidFileItem>>>>; override;
public public
function ParseFileName(const FileName: string): TAuraDataFile; override; function ParseFileName(const FileName: string): TDataFile; override;
end; end;
// File record for cTrader M1 data // File record for cTrader M1 data
TAuraM1FileItem = packed record TM1FileItem = packed record
OADateTime: Double; OADateTime: Double;
Open: Int64; Open: Int64;
High: Int64; High: Int64;
@@ -130,14 +130,14 @@ type
// File server for cTrader M1 data. Note that the generic type is TOhlcItem, // File server for cTrader M1 data. Note that the generic type is TOhlcItem,
// as the server converts the file data into standard OHLC items. // as the server converts the file data into standard OHLC items.
TAuraM1FileServer = class(TAuraDataServer<TOhlcItem>) TM1FileServer = class(TDataServer<TOhlcItem>)
private private
class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>; class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>; class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
protected protected
function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>; override; function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>; override;
public public
function ParseFileName(const FileName: string): TAuraDataFile; override; function ParseFileName(const FileName: string): TDataFile; override;
end; end;
implementation implementation
@@ -153,9 +153,9 @@ const
// The number of records per data chunk when loading files. // The number of records per data chunk when loading files.
ChunkSize = 512; ChunkSize = 512;
{ TAuraDataFile } { TDataFile }
constructor TAuraDataFile.Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer; const ABasename: String); constructor TDataFile.Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer; const ABasename: String);
begin begin
FPath := APath; FPath := APath;
FSymbol := ASymbol; FSymbol := ASymbol;
@@ -165,24 +165,24 @@ begin
FBasename := ABasename; FBasename := ABasename;
end; end;
function TAuraDataFile.GetBaseFileName: string; function TDataFile.GetBaseFileName: string;
begin begin
Result := FBasename; Result := FBasename;
end; end;
function TAuraDataFile.GetFullFileName: string; function TDataFile.GetFullFileName: string;
begin begin
Result := TPath.Combine(FPath, GetBaseFileName + FExtension); Result := TPath.Combine(FPath, GetBaseFileName + FExtension);
end; end;
function TAuraDataFile.GetIsValid: Boolean; function TDataFile.GetIsValid: Boolean;
begin begin
Result := (FSymbol <> '') and (FYear > 0); Result := (FSymbol <> '') and (FYear > 0);
end; end;
{ TAuraDataServer<T> } { TDataServer<T> }
constructor TAuraDataServer<T>.Create(const APath: String); constructor TDataServer<T>.Create(const APath: String);
begin begin
inherited Create; inherited Create;
FPath := APath; FPath := APath;
@@ -197,38 +197,38 @@ begin
); );
end; end;
destructor TAuraDataServer<T>.Destroy; destructor TDataServer<T>.Destroy;
begin begin
ClearCache; ClearCache;
FCachedFiles.Free; FCachedFiles.Free;
inherited Destroy; inherited Destroy;
end; end;
procedure TAuraDataServer<T>.AfterConstruction; procedure TDataServer<T>.AfterConstruction;
begin begin
inherited; inherited;
UpdateSymbols; UpdateSymbols;
end; end;
procedure TAuraDataServer<T>.ClearCache; procedure TDataServer<T>.ClearCache;
begin begin
FCachedFiles.Clear; FCachedFiles.Clear;
end; end;
procedure TAuraDataServer<T>.UpdateSymbols; procedure TDataServer<T>.UpdateSymbols;
begin begin
FSymbols := FSymbols :=
TFuture<TDictionary<String, TArray<TAuraDataFile>>>.Construct( TFuture<TDictionary<String, TArray<TDataFile>>>.Construct(
FSymbols.Done, FSymbols.Done,
function: TDictionary<String, TArray<TAuraDataFile>> function: TDictionary<String, TArray<TDataFile>>
var var
fileNames: TArray<string>; fileNames: TArray<string>;
tempSymbolMap: TDictionary<String, TList<TAuraDataFile>>; tempSymbolMap: TDictionary<String, TList<TDataFile>>;
dataFile: TAuraDataFile; dataFile: TDataFile;
symbolList: TList<TAuraDataFile>; symbolList: TList<TDataFile>;
begin begin
Result := TDictionary<String, TArray<TAuraDataFile>>.Create; Result := TDictionary<String, TArray<TDataFile>>.Create;
tempSymbolMap := TDictionary<String, TList<TAuraDataFile>>.Create; tempSymbolMap := TDictionary<String, TList<TDataFile>>.Create;
try try
if not TDirectory.Exists(FPath) then if not TDirectory.Exists(FPath) then
exit; exit;
@@ -241,7 +241,7 @@ begin
begin begin
if not tempSymbolMap.TryGetValue(dataFile.Symbol, symbolList) then if not tempSymbolMap.TryGetValue(dataFile.Symbol, symbolList) then
begin begin
symbolList := TList<TAuraDataFile>.Create; symbolList := TList<TDataFile>.Create;
tempSymbolMap.Add(dataFile.Symbol, symbolList); tempSymbolMap.Add(dataFile.Symbol, symbolList);
end; end;
symbolList.Add(dataFile); symbolList.Add(dataFile);
@@ -252,8 +252,8 @@ begin
begin begin
symbolList := kvp.Value; symbolList := kvp.Value;
symbolList.Sort( symbolList.Sort(
TComparer<TAuraDataFile>.Construct( TComparer<TDataFile>.Construct(
function(const Left, Right: TAuraDataFile): Integer function(const Left, Right: TDataFile): Integer
begin begin
if Left.Year < Right.Year then if Left.Year < Right.Year then
Result := -1 Result := -1
@@ -277,37 +277,37 @@ begin
FSymbols.Manage; FSymbols.Manage;
end; end;
function TAuraDataServer<T>.EnumerateSymbols: TFuture<TArray<String>>; function TDataServer<T>.EnumerateSymbols: TFuture<TArray<String>>;
begin begin
Result := Result :=
FSymbols.Chain<TArray<String>>( FSymbols.Chain<TArray<String>>(
function(const Symbols: TDictionary<String, TArray<TAuraDataFile>>): TArray<String> begin Result := Symbols.Keys.ToArray; end function(const Symbols: TDictionary<String, TArray<TDataFile>>): TArray<String> begin Result := Symbols.Keys.ToArray; end
); );
end; end;
function TAuraDataServer<T>.FindFirstFile(const Symbol: string): TFuture<TAuraDataFile>; function TDataServer<T>.FindFirstFile(const Symbol: string): TFuture<TDataFile>;
begin begin
var symName := Symbol; var symName := Symbol;
Result := Result :=
FSymbols.Chain<TAuraDataFile>( FSymbols.Chain<TDataFile>(
function(const Symbols: TDictionary<String, TArray<TAuraDataFile>>): TAuraDataFile function(const Symbols: TDictionary<String, TArray<TDataFile>>): TDataFile
var var
symbolFiles: TArray<TAuraDataFile>; symbolFiles: TArray<TDataFile>;
begin begin
if Symbols.TryGetValue(symName, symbolFiles) and (Length(symbolFiles) > 0) then if Symbols.TryGetValue(symName, symbolFiles) and (Length(symbolFiles) > 0) then
Result := symbolFiles[0] Result := symbolFiles[0]
else else
Result := Default(TAuraDataFile); Result := Default(TDataFile);
end end
); );
end; end;
function TAuraDataServer<T>.GetNextFile(const Curr: TAuraDataFile): TAuraDataFile; function TDataServer<T>.GetNextFile(const Curr: TDataFile): TDataFile;
var var
allSymbolFiles: TArray<TAuraDataFile>; allSymbolFiles: TArray<TDataFile>;
foundIndex: Integer; foundIndex: Integer;
begin begin
Result := Default(TAuraDataFile); Result := Default(TDataFile);
if not Curr.IsValid or not FSymbols.Done.IsSet then if not Curr.IsValid or not FSymbols.Done.IsSet then
exit; exit;
@@ -330,12 +330,12 @@ begin
end; end;
end; end;
function TAuraDataServer<T>.GetPath: String; function TDataServer<T>.GetPath: String;
begin begin
Result := FPath; Result := FPath;
end; end;
function TAuraDataServer<T>.ProcessData( function TDataServer<T>.ProcessData(
const Symbol: String; const Symbol: String;
const Terminated: TState; const Terminated: TState;
const Processor: IConsumer<TArray<TDataPoint<T>>> const Processor: IConsumer<TArray<TDataPoint<T>>>
@@ -347,13 +347,13 @@ begin
Result := Result :=
FindFirstFile(Symbol) FindFirstFile(Symbol)
.Chain( .Chain(
function(const FirstFileInfo: TAuraDataFile): TState function(const FirstFileInfo: TDataFile): TState
begin begin
Result := ProcessFile(FirstFileInfo, LoadDataFile(FirstFileInfo), cTerminated, cProc); Result := ProcessFile(FirstFileInfo, LoadDataFile(FirstFileInfo), cTerminated, cProc);
end); end);
end; end;
function TAuraDataServer<T>.LoadDataFile(const DataFile: TAuraDataFile): TFuture<TArray<TArray<TDataPoint<T>>>>; function TDataServer<T>.LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TArray<TDataPoint<T>>>>;
begin begin
if DataFile.IsValid then if DataFile.IsValid then
Result := FCachedFiles.GetOrAdd(DataFile.GetFullFileName) Result := FCachedFiles.GetOrAdd(DataFile.GetFullFileName)
@@ -361,7 +361,7 @@ begin
Result := TFuture<TArray<TArray<TDataPoint<T>>>>.Construct(TArray<TArray<TDataPoint<T>>>(nil)); Result := TFuture<TArray<TArray<TDataPoint<T>>>>.Construct(TArray<TArray<TDataPoint<T>>>(nil));
end; end;
function TAuraDataServer<T>.ProcessChunks( function TDataServer<T>.ProcessChunks(
const DataChunks: TArray<TArray<TDataPoint<T>>>; const DataChunks: TArray<TArray<TDataPoint<T>>>;
Terminated: TState; Terminated: TState;
Processor: IConsumer<TArray<TDataPoint<T>>> Processor: IConsumer<TArray<TDataPoint<T>>>
@@ -387,8 +387,8 @@ begin
end end
end; end;
function TAuraDataServer<T>.ProcessFile( function TDataServer<T>.ProcessFile(
FileInfo: TAuraDataFile; FileInfo: TDataFile;
const DataFile: TFuture<TArray<TArray<TDataPoint<T>>>>; const DataFile: TFuture<TArray<TArray<TDataPoint<T>>>>;
const Terminated: TState; const Terminated: TState;
Processor: IConsumer<TArray<TDataPoint<T>>> Processor: IConsumer<TArray<TDataPoint<T>>>
@@ -417,11 +417,11 @@ begin
); );
end; end;
{ TAuraTABFileServer } { TAskBidFileServer }
function TAuraTABFileServer.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>; function TAskBidFileServer.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TAskBidFileItem>>>>;
begin begin
Result := TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>.Null; Result := TFuture<TArray<TArray<TDataPoint<TAskBidFileItem>>>>.Null;
if not TFile.Exists(FileName) then if not TFile.Exists(FileName) then
exit; exit;
@@ -432,8 +432,8 @@ begin
Result := Result :=
TFuture<TBytes> TFuture<TBytes>
.Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end) .Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
.Chain<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>( .Chain<TArray<TArray<TDataPoint<TAskBidFileItem>>>>(
function(bytes: TBytes): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>> function(bytes: TBytes): TArray<TArray<TDataPoint<TAskBidFileItem>>>
begin begin
var compMemoryStream := TBytesStream.Create(bytes); var compMemoryStream := TBytesStream.Create(bytes);
try try
@@ -447,9 +447,9 @@ begin
else else
begin begin
Result := Result :=
TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>.Construct( TFuture<TArray<TArray<TDataPoint<TAskBidFileItem>>>>.Construct(
Gate.Enqueue, Gate.Enqueue,
function: TArray<TArray<TDataPoint<TAuraAskBidFileItem>>> function: TArray<TArray<TDataPoint<TAskBidFileItem>>>
begin begin
if TFile.Exists(capFileName) then if TFile.Exists(capFileName) then
begin begin
@@ -465,13 +465,13 @@ begin
end; end;
end; end;
function TAuraTABFileServer.ParseFileName(const FileName: string): TAuraDataFile; function TAskBidFileServer.ParseFileName(const FileName: string): TDataFile;
var var
fileNameNoPath, nameForParsing, baseName, ext, path, symbol: string; fileNameNoPath, nameForParsing, baseName, ext, path, symbol: string;
year, month: Integer; year, month: Integer;
parts: TArray<string>; parts: TArray<string>;
begin begin
Result := Default(TAuraDataFile); Result := Default(TDataFile);
path := TPath.GetDirectoryName(FileName); path := TPath.GetDirectoryName(FileName);
fileNameNoPath := TPath.GetFileName(FileName); fileNameNoPath := TPath.GetFileName(FileName);
@@ -509,10 +509,10 @@ begin
if symbol = '' then if symbol = '' then
exit; exit;
Result := TAuraDataFile.Create(path, symbol, fileExt, year, month, baseName); Result := TDataFile.Create(path, symbol, fileExt, year, month, baseName);
end; end;
class function TAuraTABFileServer.ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>; class function TAskBidFileServer.ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAskBidFileItem>>>;
var var
decompressionStream: TStream; decompressionStream: TStream;
localHeader: TZipHeader; localHeader: TZipHeader;
@@ -551,11 +551,11 @@ begin
end; end;
end; end;
class function TAuraTABFileServer.ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>; class function TAskBidFileServer.ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAskBidFileItem>>>;
type type
TFileRecord = packed record TFileRecord = packed record
TimeStamp: TDateTime; TimeStamp: TDateTime;
Data: TAuraAskBidFileItem; Data: TAskBidFileItem;
end; end;
var var
fileSize: Int64; fileSize: Int64;
@@ -592,15 +592,15 @@ begin
raise EReadError.CreateFmt('Read error. Expected %d bytes, read %d.', [SizeOf(TFileRecord), bytesRead]); raise EReadError.CreateFmt('Read error. Expected %d bytes, read %d.', [SizeOf(TFileRecord), bytesRead]);
// Corrected syntax: Assign the newly created record instance. // Corrected syntax: Assign the newly created record instance.
Result[chunkIndex][indexInChunk] := TDataPoint<TAuraAskBidFileItem>.Create(rec.TimeStamp, rec.Data); Result[chunkIndex][indexInChunk] := TDataPoint<TAskBidFileItem>.Create(rec.TimeStamp, rec.Data);
Inc(indexInChunk); Inc(indexInChunk);
end; end;
end; end;
end; end;
{ TAuraM1FileServer } { TM1FileServer }
function TAuraM1FileServer.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>; function TM1FileServer.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>;
begin begin
Result := TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>.Null; Result := TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>.Null;
@@ -628,13 +628,13 @@ begin
end; end;
end; end;
function TAuraM1FileServer.ParseFileName(const FileName: string): TAuraDataFile; function TM1FileServer.ParseFileName(const FileName: string): TDataFile;
var var
path, fileNameNoPath, baseName, ext, symbol: string; path, fileNameNoPath, baseName, ext, symbol: string;
year, month: Integer; year, month: Integer;
parts, dateParts: TArray<string>; parts, dateParts: TArray<string>;
begin begin
Result := Default(TAuraDataFile); Result := Default(TDataFile);
path := TPath.GetDirectoryName(FileName); path := TPath.GetDirectoryName(FileName);
fileNameNoPath := TPath.GetFileName(FileName); fileNameNoPath := TPath.GetFileName(FileName);
@@ -670,11 +670,11 @@ begin
if symbol = '' then if symbol = '' then
exit; exit;
// The TAuraDataFile only represents the start date of the file batch. // The TDataFile only represents the start date of the file batch.
Result := TAuraDataFile.Create(path, symbol, ext, year, month, baseName); Result := TDataFile.Create(path, symbol, ext, year, month, baseName);
end; end;
class function TAuraM1FileServer.ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>; class function TM1FileServer.ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
var var
decompressionStream: TStream; decompressionStream: TStream;
localHeader: TZipHeader; localHeader: TZipHeader;
@@ -725,9 +725,9 @@ begin
end; end;
end; end;
class function TAuraM1FileServer.ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>; class function TM1FileServer.ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
type type
TFileRecord = TAuraM1FileItem; TFileRecord = TM1FileItem;
var var
fileSize: Int64; fileSize: Int64;
totalRecordCount, bytesRead, chunkIndex, indexInChunk, recordIndex: Integer; totalRecordCount, bytesRead, chunkIndex, indexInChunk, recordIndex: Integer;