Brummel/MycLib
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

AuraTrader Sample

Browse Source
This commit is contained in:
Michael Schimmel
2025-06-17 21:49:09 +02:00
parent f81337c98d
commit a9aff8c41a
13 changed files with 2093 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Src/Myc.Trade.DataStream.pas
+264 -269
View File
@@ -30,34 +30,6 @@ uses
Myc.Core.FileCache;
type
// Represents metadata for a single data file.
TDataFile = record
private
FExtension: String;
FPath: String;
FSymbol: String;
FYear: Integer;
FMonth: Integer;
function GetIsValid: Boolean;
public
constructor Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer);
function GetBaseFileName: string;
function GetFullFileName: string;
// Gets the next consecutive data file, if it exists on disk.
function GetNextFile: TDataFile;
class function ParseFileName(const FileName: string): TDataFile; static;
// Scans a directory to find the oldest file for a specific symbol.
class function FindFirst(const DirectoryPath, Symbol: string): TDataFile; static;
// Scans a directory and returns the oldest file found for each symbol.
class function FindOldestPerSymbol(const DirectoryPath: string): TArray<TDataFile>; static;
property IsValid: Boolean read GetIsValid;
property Extension: String read FExtension;
property Path: String read FPath;
property Symbol: String read FSymbol;
property Year: Integer read FYear;
property Month: Integer read FMonth;
end;
// Represents a generic data stream capable of providing sequential data chunks.
// IsHistory:
// - true, if this stream is a history stream. Once HasData becomes false, it reached it's end and will not provide more data.
@@ -99,34 +71,66 @@ type
['{1F8E5A9D-E92A-44C1-9F3F-C4B82A6E94B3}']
function CreateStream(const Symbol: String): IDataStream<T>;
procedure ClearCache;
function EnumerateSymbols: TArray<String>;
function EnumerateSymbols: TFuture<TArray<String>>;
end;
// Aura files
// Represents metadata for a single data file.
TAuraDataFile = record
private
FExtension: String;
FPath: String;
FSymbol: String;
FYear: Integer;
FMonth: Integer;
function GetIsValid: Boolean;
public
constructor Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer);
function GetBaseFileName: string;
function GetFullFileName: string;
// Gets the next consecutive data file, if it exists on disk.
function GetNextFile: TAuraDataFile;
property IsValid: Boolean read GetIsValid;
property Extension: String read FExtension;
property Path: String read FPath;
property Symbol: String read FSymbol;
property Year: Integer read FYear;
property Month: Integer read FMonth;
end;
// Generic server for loading and managing sequential time-series data from files.
TAuraDataServer<T: record> = class(TInterfacedObject, IDataServer<T>)
private
FCachedFiles: TDataFileCache<TArray<TDataPoint<T>>>;
FPath: String;
function DoLoad(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
FSymbols: TFuture<TDictionary<String, TAuraDataFile>>;
function GetPath: String;
// Used by cache to actually load an uncached file.
function DoLoad(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
strict private
class var
FLoadGate: TLatch;
protected
class function ReadCompressedData(const InputStream: TStream): TArray<TDataPoint<T>>; static;
class function ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint<T>>; static;
// Scans a directory to find the oldest file for a specific symbol.
function FindFirstFile(const Symbol: string): TFuture<TAuraDataFile>;
function ParseFileName(const FileName: string): TAuraDataFile; virtual; abstract;
class function ReadCompressedData(const InputStream: TStream): TArray<TDataPoint<T>>; virtual; abstract;
class function ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint<T>>; virtual; abstract;
public
constructor Create(const APath: String);
destructor Destroy; override;
procedure AfterConstruction; override;
function CreateStream(const Symbol: String): IDataStream<T>; virtual; abstract;
procedure ClearCache;
function EnumerateSymbols: TArray<String>;
function LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TDataPoint<T>>>;
procedure UpdateSymbols;
function EnumerateSymbols: TFuture<TArray<String>>;
function LoadDataFile(const DataFile: TAuraDataFile): TFuture<TArray<TDataPoint<T>>>;
property Path: String read GetPath;
end;
@@ -135,7 +139,7 @@ type
private
type
TDataStream = record
FileInfo: TDataFile;
FileInfo: TAuraDataFile;
Data: TFuture<TArray<TDataPoint<T>>>;
end;
private
@@ -162,9 +166,14 @@ type
// Aura tick data file Ask-Bid
TAuraTABFileServer = class(TAuraDataServer<TAskBidItem>)
protected
function ParseFileName(const FileName: string): TAuraDataFile; override;
// Scans a directory and returns the oldest file found for each symbol.
class function ReadCompressedData(const InputStream: TStream): TArray<TDataPoint<TAskBidItem>>; override;
class function ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint<TAskBidItem>>; override;
public
function CreateStream(const Symbol: String): IDataStream<TAskBidItem>; override;
function LoadDataSeries(const InitialFile: TDataFile): TFuture<TArray<TDataPoint<TAskBidItem>>>;
function LoadDataSeries(const InitialFile: TAuraDataFile): TFuture<TArray<TDataPoint<TAskBidItem>>>;
end;
implementation
@@ -175,9 +184,9 @@ uses
System.StrUtils,
Myc.TaskManager;
{ TDataFile }
{ TAuraDataFile }
constructor TDataFile.Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer);
constructor TAuraDataFile.Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer);
begin
FPath := APath;
FSymbol := ASymbol;
@@ -186,28 +195,28 @@ begin
FMonth := AMonth;
end;
function TDataFile.GetBaseFileName: string;
function TAuraDataFile.GetBaseFileName: string;
begin
Result := Format('%s_%.4d_%.2d', [FSymbol, FYear, FMonth]);
end;
function TDataFile.GetFullFileName: string;
function TAuraDataFile.GetFullFileName: string;
begin
Result := TPath.Combine(FPath, GetBaseFileName + FExtension);
end;
function TDataFile.GetIsValid: Boolean;
function TAuraDataFile.GetIsValid: Boolean;
begin
Result := (FSymbol <> '') and (FYear > 0);
end;
function TDataFile.GetNextFile: TDataFile;
function TAuraDataFile.GetNextFile: TAuraDataFile;
var
nextMonth, nextYear: Integer;
nextFile: TDataFile;
nextFile: TAuraDataFile;
begin
if not IsValid then
exit(Default(TDataFile));
exit(Default(TAuraDataFile));
nextMonth := FMonth + 1;
nextYear := FYear;
@@ -218,137 +227,19 @@ begin
end;
// Probe for zipped file first
nextFile := TDataFile.Create(FPath, FSymbol, '.tab_zip', nextYear, nextMonth);
nextFile := TAuraDataFile.Create(FPath, FSymbol, '.tab_zip', nextYear, nextMonth);
if TFile.Exists(nextFile.GetFullFileName) then
exit(nextFile);
// Probe for uncompressed file
nextFile := TDataFile.Create(FPath, FSymbol, '.tab', nextYear, nextMonth);
nextFile := TAuraDataFile.Create(FPath, FSymbol, '.tab', nextYear, nextMonth);
if TFile.Exists(nextFile.GetFullFileName) then
exit(nextFile);
// No next file found
Result := Default(TDataFile);
Result := Default(TAuraDataFile);
end;
class function TDataFile.ParseFileName(const FileName: string): TDataFile;
var
fileNameNoPath, nameForParsing, baseName, ext, path, symbol: string;
year, month: Integer;
parts: TArray<string>;
begin
Result := Default(TDataFile);
path := TPath.GetDirectoryName(FileName);
fileNameNoPath := TPath.GetFileName(FileName);
nameForParsing := fileNameNoPath;
ext := '';
if nameForParsing.EndsWith('_zip', True) then
begin
ext := '_zip';
nameForParsing := nameForParsing.Substring(0, nameForParsing.Length - 4);
end;
var fileExt := TPath.GetExtension(nameForParsing) + ext;
baseName := TPath.GetFileNameWithoutExtension(nameForParsing);
parts := baseName.Split(['_']);
if Length(parts) < 3 then
exit;
if not TryStrToInt(parts[High(parts)], month) then
exit;
if (month < 1) or (month > 12) then
exit;
if not TryStrToInt(parts[High(parts) - 1], year) then
exit;
if year <= 0 then
exit;
symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
if symbol = '' then
exit;
Result := TDataFile.Create(path, symbol, fileExt, year, month);
end;
class function TDataFile.FindFirst(const DirectoryPath, Symbol: string): TDataFile;
var
fileNames: TArray<string>;
currentFileName: string;
parsedFile: TDataFile;
begin
Result := Default(TDataFile);
if not TDirectory.Exists(DirectoryPath) then
exit;
fileNames := TDirectory.GetFiles(DirectoryPath);
for currentFileName in fileNames do
begin
parsedFile := TDataFile.ParseFileName(currentFileName);
if parsedFile.IsValid then
begin
if SameText(parsedFile.Symbol, Symbol) then
begin
// If it's the first match or older than the current result, update.
if (not Result.IsValid) or
(parsedFile.Year < Result.Year) or
((parsedFile.Year = Result.Year) and (parsedFile.Month < Result.Month)) then
begin
Result := parsedFile;
end;
end;
end;
end;
end;
class function TDataFile.FindOldestPerSymbol(const DirectoryPath: string): TArray<TDataFile>;
var
oldestFilesPerSymbol: TDictionary<string, TDataFile>;
fileNames: TArray<string>;
currentFile: string;
dataFile: TDataFile;
trackedInfo: TDataFile;
begin
oldestFilesPerSymbol := TDictionary<string, TDataFile>.Create;
try
if not TDirectory.Exists(DirectoryPath) then
exit(nil);
fileNames := TDirectory.GetFiles(DirectoryPath);
for currentFile in fileNames do
begin
dataFile := TDataFile.ParseFileName(currentFile);
if dataFile.IsValid then
begin
if oldestFilesPerSymbol.TryGetValue(dataFile.Symbol, trackedInfo) then
begin
if (dataFile.Year < trackedInfo.Year) or ((dataFile.Year = trackedInfo.Year) and (dataFile.Month < trackedInfo.Month)) then
begin
oldestFilesPerSymbol.AddOrSetValue(dataFile.Symbol, dataFile);
end;
end
else
begin
oldestFilesPerSymbol.Add(dataFile.Symbol, dataFile);
end;
end;
end;
Result := oldestFilesPerSymbol.Values.ToArray;
finally
oldestFilesPerSymbol.Free;
end;
end;
{ TAuraDataServer<T> }
constructor TAuraDataServer<T>.Create(const APath: String);
@@ -368,27 +259,17 @@ begin
inherited Destroy;
end;
procedure TAuraDataServer<T>.AfterConstruction;
begin
inherited;
UpdateSymbols;
end;
procedure TAuraDataServer<T>.ClearCache;
begin
FCachedFiles.Clear;
end;
function TAuraDataServer<T>.EnumerateSymbols: TArray<String>;
var
dataFiles: TArray<TDataFile>;
i: Integer;
begin
dataFiles := TDataFile.FindOldestPerSymbol(FPath);
SetLength(Result, Length(dataFiles));
for i := 0 to High(dataFiles) do
Result[i] := dataFiles[i].Symbol;
end;
function TAuraDataServer<T>.GetPath: String;
begin
Result := FPath;
end;
function TAuraDataServer<T>.DoLoad(const FileName: string): TFuture<TArray<TDataPoint<T>>>;
begin
Result := TFuture<TArray<TDataPoint<T>>>.Null;
@@ -401,9 +282,7 @@ begin
begin
Result :=
TFuture<TBytes>
.Construct(
TLatch.Enqueue(FLoadGate),
function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
.Construct(FLoadGate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
.Chain<TArray<TDataPoint<T>>>(
function(bytes: TBytes): TArray<TDataPoint<T>>
begin
@@ -420,7 +299,7 @@ begin
begin
Result :=
TFuture<TArray<TDataPoint<T>>>.Construct(
TLatch.Enqueue(FLoadGate),
FLoadGate.Enqueue,
function: TArray<TDataPoint<T>>
begin
if TFile.Exists(capFileName) then
@@ -442,81 +321,82 @@ begin
end;
end;
function TAuraDataServer<T>.LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TDataPoint<T>>>;
procedure TAuraDataServer<T>.UpdateSymbols;
begin
FSymbols :=
TFuture<TDictionary<String, TAuraDataFile>>.Construct(
FSymbols.Done,
function: TDictionary<String, TAuraDataFile>
var
fileNames: TArray<string>;
currentFile: string;
dataFile: TAuraDataFile;
trackedInfo: TAuraDataFile;
begin
Result := TDictionary<string, TAuraDataFile>.Create;
if not TDirectory.Exists(FPath) then
exit;
fileNames := TDirectory.GetFiles(FPath);
for currentFile in fileNames do
begin
dataFile := ParseFileName(currentFile);
if dataFile.IsValid then
begin
if Result.TryGetValue(dataFile.Symbol, trackedInfo) then
begin
if (dataFile.Year < trackedInfo.Year)
or ((dataFile.Year = trackedInfo.Year) and (dataFile.Month < trackedInfo.Month)) then
begin
Result.AddOrSetValue(dataFile.Symbol, dataFile);
end;
end
else
begin
Result.Add(dataFile.Symbol, dataFile);
end;
end;
end;
end
);
FSymbols.Manage;
end;
function TAuraDataServer<T>.EnumerateSymbols: TFuture<TArray<String>>;
var
dataFiles: TArray<TAuraDataFile>;
i: Integer;
begin
Result :=
FSymbols.Chain<TArray<String>>(
function(Symbols: TDictionary<String, TAuraDataFile>): TArray<String> begin Result := Symbols.Keys.ToArray; end
);
end;
function TAuraDataServer<T>.FindFirstFile(const Symbol: string): TFuture<TAuraDataFile>;
begin
var symName := Symbol;
Result :=
FSymbols.Chain<TAuraDataFile>(
function(Symbols: TDictionary<String, TAuraDataFile>): TAuraDataFile
begin
if not Symbols.TryGetValue(symName, Result) then
Result := Default(TAuraDataFile);
end
);
end;
function TAuraDataServer<T>.GetPath: String;
begin
Result := FPath;
end;
function TAuraDataServer<T>.LoadDataFile(const DataFile: TAuraDataFile): TFuture<TArray<TDataPoint<T>>>;
begin
Result := FCachedFiles.GetOrAdd(DataFile.GetFullFileName);
end;
class function TAuraDataServer<T>.ReadCompressedData(const InputStream: TStream): TArray<TDataPoint<T>>;
var
decompressionStream: TStream;
localHeader: TZipHeader;
entryIndex, i: Integer;
zipFileInstance: TZipFile;
begin
SetLength(Result, 0);
decompressionStream := nil;
zipFileInstance := nil;
try
InputStream.Position := 0;
zipFileInstance := TZipFile.Create;
zipFileInstance.Open(InputStream, TZipMode.zmRead);
if zipFileInstance.FileCount = 0 then
exit;
entryIndex := -1;
for i := 0 to zipFileInstance.FileCount - 1 do
if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
begin
entryIndex := i;
break;
end;
if entryIndex = -1 then
entryIndex := 0;
zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
if not Assigned(decompressionStream) then
exit;
Result := ReadUncompressedData(decompressionStream);
finally
decompressionStream.Free;
zipFileInstance.Free;
end;
end;
class function TAuraDataServer<T>.ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint<T>>;
type
TFileRecord = packed record
TimeStamp: TDateTime;
Data: T;
end;
var
fileSize: Int64;
recordCount, bytesRead: Integer;
rec: TFileRecord;
begin
SetLength(Result, 0);
InputStream.Position := 0;
fileSize := InputStream.Size;
if (fileSize = 0) or ((fileSize mod SizeOf(TFileRecord)) <> 0) then
exit;
recordCount := fileSize div SizeOf(TFileRecord);
if recordCount > 0 then
begin
SetLength(Result, recordCount);
for var i := 0 to High(Result) do
begin
bytesRead := InputStream.Read(rec, SizeOf(TFileRecord));
if bytesRead <> SizeOf(TFileRecord) then
raise EReadError.CreateFmt('Read error. Expected %d bytes, read %d.', [fileSize, bytesRead]);
Result[i].Create(rec.TimeStamp, rec.Data);
end;
end;
end;
{ TAuraFileStream<T> }
constructor TAuraFileStream<T>.Create(ADataServer: TAuraDataServer<T>; const ASymbol: String);
@@ -535,18 +415,16 @@ end;
procedure TAuraFileStream<T>.AfterConstruction;
var
firstFile: TFuture<TDataFile>;
serverPath: string;
firstFile: TFuture<TAuraDataFile>;
begin
inherited;
// We need the path from the server instance to find the file.
serverPath := (FDataServer as TAuraDataServer<T>).Path;
firstFile := TFuture<TDataFile>.Construct(function: TDataFile begin Result := TDataFile.FindFirst(serverPath, FSymbol); end);
firstFile := FDataServer.FindFirstFile(FSymbol);
FCurrent :=
firstFile.Chain<TDataStream>(
function(const FileInfo: TDataFile): TDataStream
function(const FileInfo: TAuraDataFile): TDataStream
begin
if FileInfo.IsValid then
begin
@@ -638,7 +516,7 @@ begin
FCurrent.Chain<TDataStream>(
function(const Prev: TDataStream): TDataStream
var
nextFileInfo: TDataFile;
nextFileInfo: TAuraDataFile;
begin
if Prev.FileInfo.IsValid then
begin
@@ -661,16 +539,16 @@ begin
Result := TAuraFileStream<TAskBidItem>.Create(Self, Symbol);
end;
function TAuraTABFileServer.LoadDataSeries(const InitialFile: TDataFile): TFuture<TArray<TDataPoint<TAskBidItem>>>;
function TAuraTABFileServer.LoadDataSeries(const InitialFile: TAuraDataFile): TFuture<TArray<TDataPoint<TAskBidItem>>>;
var
loadedState: TState;
loadedFiles: TArray<TFuture<TArray<TDataPoint<TAskBidItem>>>>;
liveData: TFuture<TArray<TDataPoint<TAskBidItem>>>;
tabFiles: TList<TDataFile>;
liveFile: TDataFile;
currentFileInfo: TDataFile;
tabFiles: TList<TAuraDataFile>;
liveFile: TAuraDataFile;
currentFileInfo: TAuraDataFile;
begin
tabFiles := TList<TDataFile>.Create;
tabFiles := TList<TAuraDataFile>.Create;
try
currentFileInfo := InitialFile;
while currentFileInfo.IsValid do
@@ -679,14 +557,14 @@ begin
currentFileInfo := currentFileInfo.GetNextFile;
end;
liveFile := Default(TDataFile);
liveFile := Default(TAuraDataFile);
if tabFiles.Count > 0 then
begin
var lastTabFile := tabFiles.Last;
var liveFileBaseName := Format('%s_%d_%02d.tab-live', [lastTabFile.Symbol, lastTabFile.Year, lastTabFile.Month]);
var potentialLivePath := TPath.Combine(lastTabFile.Path, liveFileBaseName);
if TFile.Exists(potentialLivePath) then
liveFile := TDataFile.ParseFileName(potentialLivePath);
liveFile := ParseFileName(potentialLivePath);
end;
var loadedFileList := TList<TFuture<TArray<TDataPoint<TAskBidItem>>>>.Create;
@@ -745,4 +623,121 @@ begin
);
end;
function TAuraTABFileServer.ParseFileName(const FileName: string): TAuraDataFile;
var
fileNameNoPath, nameForParsing, baseName, ext, path, symbol: string;
year, month: Integer;
parts: TArray<string>;
begin
Result := Default(TAuraDataFile);
path := TPath.GetDirectoryName(FileName);
fileNameNoPath := TPath.GetFileName(FileName);
nameForParsing := fileNameNoPath;
ext := '';
if nameForParsing.EndsWith('_zip', True) then
begin
ext := '_zip';
nameForParsing := nameForParsing.Substring(0, nameForParsing.Length - 4);
end;
var fileExt := TPath.GetExtension(nameForParsing) + ext;
baseName := TPath.GetFileNameWithoutExtension(nameForParsing);
parts := baseName.Split(['_']);
if Length(parts) < 3 then
exit;
if not TryStrToInt(parts[High(parts)], month) then
exit;
if (month < 1) or (month > 12) then
exit;
if not TryStrToInt(parts[High(parts) - 1], year) then
exit;
if year <= 0 then
exit;
symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
if symbol = '' then
exit;
Result := TAuraDataFile.Create(path, symbol, fileExt, year, month);
end;
class function TAuraTABFileServer.ReadCompressedData(const InputStream: TStream): TArray<TDataPoint<TAskBidItem>>;
var
decompressionStream: TStream;
localHeader: TZipHeader;
entryIndex, i: Integer;
zipFileInstance: TZipFile;
begin
SetLength(Result, 0);
decompressionStream := nil;
zipFileInstance := nil;
try
InputStream.Position := 0;
zipFileInstance := TZipFile.Create;
zipFileInstance.Open(InputStream, TZipMode.zmRead);
if zipFileInstance.FileCount = 0 then
exit;
entryIndex := -1;
for i := 0 to zipFileInstance.FileCount - 1 do
if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
begin
entryIndex := i;
break;
end;
if entryIndex = -1 then
entryIndex := 0;
zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
if not Assigned(decompressionStream) then
exit;
Result := ReadUncompressedData(decompressionStream);
finally
decompressionStream.Free;
zipFileInstance.Free;
end;
end;
class function TAuraTABFileServer.ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint<TAskBidItem>>;
type
TFileRecord = packed record
TimeStamp: TDateTime;
Data: TAskBidItem;
end;
var
fileSize: Int64;
recordCount, bytesRead: Integer;
rec: TFileRecord;
begin
SetLength(Result, 0);
InputStream.Position := 0;
fileSize := InputStream.Size;
if (fileSize = 0) or ((fileSize mod SizeOf(TFileRecord)) <> 0) then
exit;
recordCount := fileSize div SizeOf(TFileRecord);
if recordCount > 0 then
begin
SetLength(Result, recordCount);
for var i := 0 to High(Result) do
begin
bytesRead := InputStream.Read(rec, SizeOf(TFileRecord));
if bytesRead <> SizeOf(TFileRecord) then
raise EReadError.CreateFmt('Read error. Expected %d bytes, read %d.', [fileSize, bytesRead]);
Result[i].Create(rec.TimeStamp, rec.Data);
end;
end;
end;
end.