Files
MycLib/Src/Myc.Trade.DataSeries.pas
2025-06-05 22:11:54 +02:00

623 lines
23 KiB
ObjectPascal

unit Myc.Trade.DataSeries;
(* ------------------------------------------------------------------------------
This unit provides functions to load tick data (TTickData) from
binary files. The data originates from a C# application
and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
The files follow the naming convention:
For .tab files: Symbol_Year_MM.tab OR Symbol_Year_MM.tab_zip
For .tab-live files: Symbol_Year_MM.tab-live (NEVER compressed)
(MM represents the two-digit month, e.g., 01 for January, 12 for December)
If both compressed (.tab_zip) and uncompressed (.tab) versions of a .tab
file exist, the compressed version is preferred. Files ending with _zip
are expected to be ZIP archives containing the actual data file
(e.g., Symbol_Year_MM.tab).
Loading occurs in two phases:
1. All .tab files (regular or _zip) are processed chronologically by
month and year. Unique ticks are loaded, and the timestamp of the
latest tick across all .tab files is determined (overallLastTabTickTime).
Assumes timestamps are unique across all .tab files.
2. All .tab-live files (which are never compressed), for months/years
corresponding to found .tab files, are processed. Ticks from a
.tab-live file are only integrated if their timestamp is strictly later
than overallLastTabTickTime. Assumes such ticks are globally unique.
If a .tab-live file results in no new data being added under these rules,
it is deleted after processing.
The loading function automatically detects subsequent monthly/yearly files.
Main Function:
LoadTickDataSeries - Loads a series of tick data files.
------------------------------------------------------------------------------ *)
interface
uses
System.SysUtils,
System.Classes,
System.Generics.Collections,
System.Generics.Defaults,
System.IOUtils,
Myc.Futures,
Myc.Signals;
type
TDataSeries<T: record> = class
type
TDataPoint = packed record
OADateTime: Double;
Data: T;
end;
strict private
type
TCachedFile = record
Name: String;
Age: TDateTime;
LastUsed: TDateTime;
Data: TFuture<TArray<TDataPoint>>;
end;
class var
FCachedFiles: TDictionary<String, TCachedFile>;
class constructor CreateClass;
class destructor DestroyClass;
protected
class function ReadCompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
class function ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
public
class function LoadDataFile(var LoadGate: TLatch; const FileName: string): TFuture<TArray<TDataPoint>>; static;
class function LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint>>; static;
end;
TAskBidItem = packed record
Ask: Single;
Bid: Single;
end;
TAskBid = class(TDataSeries<TAskBidItem>)
type
TTick = TDataSeries<TAskBidItem>.TDataPoint;
end;
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue, MonthValue: Integer): Boolean;
// Finds the full filename of the oldest (=first) file for each symbol in the path.
function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
var
IsMemoryLow: TFunc<Boolean>;
implementation
uses
System.Zip,
System.Math,
Myc.TaskManager;
// TryParseFileName parses filenames like "Symbol_Year_MM.Extension" or "Symbol_Year_MM.Extension_zip"
// to extract Path, Symbol, Year, and Month.
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue, MonthValue: Integer): Boolean;
var
fileNameNoPath: string;
nameForParsing: string;
parts: TArray<string>;
baseName: string;
begin
Result := False; // Default to failure
PathValue := ''; // Initialize all out parameters
SymbolValue := '';
YearValue := 0;
MonthValue := 0;
PathValue := TPath.GetDirectoryName(FileName);
fileNameNoPath := TPath.GetFileName(FileName);
nameForParsing := fileNameNoPath;
// Handle cases like "Symbol_Year_MM.Extension_zip" by removing the trailing "_zip" before extension removal
if nameForParsing.EndsWith('_zip', True) then
begin
// Get the part before "_zip"
baseName := nameForParsing.Substring(0, nameForParsing.Length - 4);
// Get the extension of the original name (e.g. ".csv_zip")
// and check if baseName still contains an extension that needs to be removed by GetFileNameWithoutExtension
if TPath.GetExtension(baseName) <> '' then
begin
nameForParsing := baseName;
end
else
begin
// If baseName (e.g. "EURUSD_2023_01") has no extension, it is the correct parsing base
// However, TPath.GetFileNameWithoutExtension would also work correctly here,
// but this handles a potential case where a file might be named "Symbol_Year_MM_zip" (no dot before zip)
end;
end;
// Remove the actual extension (e.g., ".csv", ".dat") from "Symbol_Year_MM.Extension" or the modified "Symbol_Year_MM.Extension" part of "_zip"
nameForParsing := TPath.GetFileNameWithoutExtension(nameForParsing); // e.g., "EURUSD_2023_01"
parts := nameForParsing.Split(['_']);
if Length(parts) < 3 then // Need at least Symbol_Year_Month
exit;
// Month is the last part.
if not TryStrToInt(parts[High(parts)], MonthValue) then
begin
exit;
end;
if (MonthValue < 1) or (MonthValue > 12) then // Validate month range
begin
MonthValue := 0; // Set to invalid if out of range
exit;
end;
// Year is the second to last part.
if not TryStrToInt(parts[High(parts) - 1], YearValue) then
begin
MonthValue := 0; // Also invalidate month if year parsing fails
exit;
end;
// Ensure YearValue is plausible, e.g. not 0 or negative, adjust as needed for your data's year range
if YearValue <= 0 then
begin
YearValue := 0;
MonthValue := 0;
exit;
end;
// Symbol consists of all parts before Year and Month.
SymbolValue := string.Join('_', Copy(parts, 0, Length(parts) - 2));
if SymbolValue = '' then // Ensure symbol is not empty
begin
// Reset all out parameters on failure before exit
PathValue := '';
YearValue := 0;
MonthValue := 0;
exit;
end;
Result := True;
end;
// Finds the oldest file for each symbol in the given directory.
// The file extension is ignored for the purpose of identifying the symbol, year, and month.
function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
type
TTrackedFileInfo = record
FullFileName: string;
Year: Integer;
Month: Integer;
end;
var
oldestFilesPerSymbol: TDictionary<string, TTrackedFileInfo>;
fileNames: TArray<string>;
currentFile: string;
parsedPath: string;
parsedSymbol: string;
parsedYear: Integer;
parsedMonth: Integer;
trackedInfo: TTrackedFileInfo;
begin
oldestFilesPerSymbol := TDictionary<string, TTrackedFileInfo>.Create; // Dictionary to track the oldest file per symbol
try
if not TDirectory.Exists(DirectoryPath) then
begin
// Or raise an exception, depending on desired error handling
exit;
end;
fileNames := TDirectory.GetFiles(DirectoryPath); // Get all files in the directory
for currentFile in fileNames do
begin
// Attempt to parse the file name
if TryParseFileName(currentFile, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
begin
// Check if this symbol is already tracked or if this file is older
if oldestFilesPerSymbol.TryGetValue(parsedSymbol, trackedInfo) then
begin
// If current file's year is less, or year is same and month is less, it's older
if (parsedYear < trackedInfo.Year) or ((parsedYear = trackedInfo.Year) and (parsedMonth < trackedInfo.Month)) then
begin
trackedInfo.FullFileName := currentFile; // Store full path to the file
trackedInfo.Year := parsedYear;
trackedInfo.Month := parsedMonth;
oldestFilesPerSymbol.AddOrSetValue(parsedSymbol, trackedInfo); // Update the entry for this symbol
end;
end
else
begin
// First time we see this symbol, so it's the oldest so far
trackedInfo.FullFileName := currentFile; // Store full path to the file
trackedInfo.Year := parsedYear;
trackedInfo.Month := parsedMonth;
oldestFilesPerSymbol.Add(parsedSymbol, trackedInfo); // Add new entry for this symbol
end;
end;
end;
// Populate the result array with the full names of the oldest files
SetLength(Result, oldestFilesPerSymbol.Count);
var n := 0;
for trackedInfo in oldestFilesPerSymbol.Values do
begin
Result[n] := trackedInfo.FullFileName;
inc(n);
end;
finally
oldestFilesPerSymbol.Free; // Free the dictionary
end;
end;
class destructor TDataSeries<T>.CreateClass;
begin
FCachedFiles := TDictionary<String, TCachedFile>.Create;
end;
class destructor TDataSeries<T>.DestroyClass;
begin
FCachedFiles.Free;
end;
// Asynchronously reads tick data from a single specified file.
// Returns a TFuture that will provide an array of TAskBidSeries.
class function TDataSeries<T>.LoadDataFile(var LoadGate: TLatch; const FileName: string): TFuture<TArray<TDataPoint>>;
var
parsedPath, parsedSymbol: string;
parsedYear, parsedMonth: Integer;
begin
Result := TFuture<TArray<TDataPoint>>.Null;
// Attempt to parse year and month from the filename
if not TryParseFileName(FileName, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
exit;
if Assigned(IsMemoryLow) then
begin
if IsMemoryLow then
begin
var fL := FCachedFiles.Values.ToArray;
TArray.Sort<TCachedFile>(
fL,
TComparer<TCachedFile>
.Construct(function(const Left, Right: TCachedFile): Integer begin Result := Sign(Left.LastUsed - Right.LastUsed); end)
);
for var i := 0 to High(fL) do
begin
FCachedFiles.Remove(fL[i].Name);
if not IsMemoryLow then
break;
end;
end;
var cachedFile: TCachedFile;
if FCachedFiles.TryGetValue(Filename, cachedFile) then
begin
FCachedFiles.Remove(FileName);
var age: TDateTime;
if FileAge(Filename, age, true) then
begin
if age = cachedFile.Age then
begin
cachedFile.LastUsed := Now;
FCachedFiles.Add(Filename, cachedFile);
exit(cachedFile.Data);
end;
end;
end;
end;
if FileName.EndsWith('_zip', True) then
begin
var zipFileBytes := TFile.ReadAllBytes(FileName); // Read all bytes of the .zip file
if Length(zipFileBytes) = 0 then
exit;
var capFileName := FileName;
Result :=
TFuture<TBytes>
.Construct(
TLatch.Enqueue(LoadGate),
function: TBytes
begin
Result := TFile.ReadAllBytes(capFileName); // Read all bytes of the .zip file
end)
.Chain<TArray<TDataPoint>>(
function(bytes: TBytes): TArray<TDataPoint>
begin
var zipMemoryStream := TBytesStream.Create(zipFileBytes); // Create a memory stream to hold the zip file content
try
zipMemoryStream.Position := 0;
Result := ReadCompressedData(zipMemoryStream);
finally
zipMemoryStream.Free;
end;
end);
end
else if TFile.Exists(FileName) then
begin
var capFileName := FileName;
Result :=
TFuture<TArray<TDataPoint>>.Construct(
function: TArray<TDataPoint>
begin
if TFile.Exists(capFileName) then
begin
var stream := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
try
try
Result := ReadUncompressedData(stream);
except
on E: EReadError do
begin
raise EReadError.CreateFmt('File %s: %s', [capFileName, E.Message]);
end;
end;
finally
stream.Free;
end;
end;
end
);
end;
if Assigned(IsMemoryLow) then
begin
var cachedFile: TCachedFile;
cachedFile.Name := Filename;
var age: TDateTime;
if FileAge(Filename, cachedFile.Age, true) then
begin
cachedFile.Data := Result;
cachedFile.LastUsed := Now;
FCachedFiles.Add(Filename, cachedFile);
end;
end;
end;
class function TDataSeries<T>.LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint>>;
// Local type declaration for storing file information for phase 1 processing
type
TPhase1FileEntry = record
Path: string;
Year: Integer;
Month: Integer;
end;
var
initialYear, initialMonth: Integer;
currentTabYear, currentTabMonth: Integer;
pathName, symbolName: string;
tabFileEntryRecord: TPhase1FileEntry; // Variable to construct record instances
loadedState: TState;
loadedFiles: TArray<TFuture<TArray<TDataPoint>>>;
liveData: TFuture<TArray<TDataPoint>>;
begin // Start of LoadDataSeries
if not TryParseFileName(FileName, pathName, symbolName, initialYear, initialMonth) then
begin
raise EArgumentException.CreateFmt(
'Invalid initial file name format: %s. Expected Symbol_Year_MM.Extension or Symbol_Year_MM.Extension_zip',
[FileName]);
end;
currentTabYear := initialYear;
currentTabMonth := initialMonth;
var tabBaseNameFormat: string;
var compressedTabPath, uncompressedTabPath: string;
var actualTabFileToLoad: string;
var maxTabYearFound := -1;
var maxTabMonthFound := -1;
var filesToLoadPhase1 := TList<TPhase1FileEntry>.Create; // Create list with the named record type
try
while True do
begin
tabBaseNameFormat := Format('%s_%.4d_%.2d.tab', [symbolName, currentTabYear, currentTabMonth]);
compressedTabPath := TPath.Combine(pathName, tabBaseNameFormat + '_zip');
uncompressedTabPath := TPath.Combine(pathName, tabBaseNameFormat);
actualTabFileToLoad := '';
if TFile.Exists(compressedTabPath) then
begin
actualTabFileToLoad := compressedTabPath;
end
else if TFile.Exists(uncompressedTabPath) then
begin
actualTabFileToLoad := uncompressedTabPath;
end;
if actualTabFileToLoad = '' then
break; // No more sequential .tab files
// Construct the record and add it to the list
tabFileEntryRecord.Path := actualTabFileToLoad;
tabFileEntryRecord.Year := currentTabYear;
tabFileEntryRecord.Month := currentTabMonth;
filesToLoadPhase1.Add(tabFileEntryRecord);
// Advance to the next month
Inc(currentTabMonth);
if currentTabMonth > 12 then
begin
currentTabMonth := 1;
Inc(currentTabYear);
end;
end;
// Now load and process identified .tab files
var loadedFileList := TList<TFuture<TArray<TDataPoint>>>.Create;
var loadStates := TList<TState>.Create;
try
var LoadGate: TLatch;
for var tabFileEntry: TPhase1FileEntry in filesToLoadPhase1 do // Iterate with the correct type
begin
var data := LoadDataFile(LoadGate, tabFileEntry.Path);
loadedFileList.Add(TFuture<TArray<TDataPoint>>.Create(data));
loadStates.Add(data.Done);
if tabFileEntry.Year > maxTabYearFound then
begin
maxTabYearFound := tabFileEntry.Year;
maxTabMonthFound := tabFileEntry.Month;
end
else if (tabFileEntry.Year = maxTabYearFound) and (tabFileEntry.Month > maxTabMonthFound) then
begin
maxTabMonthFound := tabFileEntry.Month;
end;
end;
var liveFileBaseName := Format('%s_%d_%02d.tab-live', [symbolName, maxTabYearFound, maxTabMonthFound]);
liveData := LoadDataFile(LoadGate, TPath.Combine(pathName, liveFileBaseName));
loadStates.Add(liveData.Done);
loadedFileList.Add(liveData);
loadedState := TState.All(loadStates.ToArray);
loadedFiles := loadedFileList.ToArray;
finally
loadStates.Free;
loadedFileList.Free;
end;
finally
filesToLoadPhase1.Free;
end;
Result :=
TFuture<TArray<TDataPoint>>.Construct(
loadedState,
function: TArray<TDataPoint>
begin
var tickList := TList<TDataPoint>.Create;
try
var totalTicks := Length(liveData.Result);
var overallLastTabTickTime := 0.0;
for var P in loadedFiles do
Inc(totalTicks, Length(P.Result));
tickList.Capacity := totalTicks;
for var P in loadedFiles do
begin
if Length(P.Result) > 0 then
begin
for var iterTickData in P.Result do
if iterTickData.OADateTime > overallLastTabTickTime then
begin
tickList.Add(iterTickData);
overallLastTabTickTime := iterTickData.OADateTime;
end;
end;
end;
Result := tickList.ToArray;
finally
tickList.Free;
end;
end
);
end;
class function TDataSeries<T>.ReadCompressedData(const InputStream: TStream): TArray<TDataPoint>;
var
decompressionStream: TStream; // Holds uncompressed data of a single zip entry
localHeader: TZipHeader;
entryIndex: Integer;
i: Integer;
zipFileInstance: TZipFile;
begin
// Initialize Result record
SetLength(Result, 0);
decompressionStream := nil;
zipFileInstance := nil;
try
InputStream.Position := 0; // Reset stream position to the beginning for TZipFile
// Open TZipFile from this memory stream
zipFileInstance := TZipFile.Create;
zipFileInstance.Open(InputStream, TZipMode.zmRead); // Open the zip from the memory stream
if zipFileInstance.FileCount = 0 then
begin
// If TZipFile opens an empty or invalid stream successfully but finds no files
exit; // FileSuccessfullyLoaded remains False
end;
entryIndex := -1;
for i := 0 to zipFileInstance.FileCount - 1 do
begin
if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
begin
entryIndex := i;
break;
end;
end;
if entryIndex = -1 then // If specific name not found, default to the first entry
begin
if zipFileInstance.FileCount > 0 then
entryIndex := 0
else
exit; // Should not happen if FileCount > 0 check passed
end;
// Decompress the specific entry from the in-memory zip file.
// TZipFile.Read creates and populates decompressionStream with uncompressed data.
zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
if not Assigned(decompressionStream) then
exit; // Exit if no valid stream could be prepared
Result := ReadUncompressedData(decompressionStream);
finally
// Ensure all potentially created objects are freed
if Assigned(decompressionStream) then // Holds uncompressed data from a zip entry
decompressionStream.Free;
if Assigned(zipFileInstance) then
zipFileInstance.Free;
end;
end;
class function TDataSeries<T>.ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint>;
var
fileSize: Int64;
recordCount: Integer;
bytesRead: Integer;
begin
// Initialize Result record
SetLength(Result, 0);
InputStream.Position := 0;
fileSize := InputStream.Size;
if fileSize = 0 then
exit;
if (fileSize mod SizeOf(TDataPoint)) <> 0 then
raise EReadError.CreateFmt('File corrupted. Size %d is not a multiple of record size %d.', [fileSize, SizeOf(TDataPoint)]);
recordCount := fileSize div SizeOf(TDataPoint);
if recordCount > 0 then
begin
SetLength(Result, recordCount);
bytesRead := InputStream.Read(Result[0], fileSize);
if bytesRead <> fileSize then
raise EReadError.CreateFmt('Read error. Expected to read %d bytes, but actually read %d bytes.', [fileSize, bytesRead]);
end;
end;
end.