Michael Schimmel
543 lines
20 KiB
ObjectPascal
543 lines
20 KiB
ObjectPascal
unit Myc.Trade.DataSeries;
|
|
|
|
(* ------------------------------------------------------------------------------
|
|
This unit provides functions to load tick data (TTickData) from
|
|
binary files. The data originates from a C# application
|
|
and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
|
|
|
|
The files follow the naming convention:
|
|
For .tab files: Symbol_Year_MM.tab OR Symbol_Year_MM.tab_zip
|
|
For .tab-live files: Symbol_Year_MM.tab-live (NEVER compressed)
|
|
(MM represents the two-digit month, e.g., 01 for January, 12 for December)
|
|
|
|
If both compressed (.tab_zip) and uncompressed (.tab) versions of a .tab
|
|
file exist, the compressed version is preferred. Files ending with _zip
|
|
are expected to be ZIP archives containing the actual data file
|
|
(e.g., Symbol_Year_MM.tab).
|
|
|
|
Loading occurs in two phases:
|
|
1. All .tab files (regular or _zip) are processed chronologically by
|
|
month and year. Unique ticks are loaded, and the timestamp of the
|
|
latest tick across all .tab files is determined (overallLastTabTickTime).
|
|
Assumes timestamps are unique across all .tab files.
|
|
2. All .tab-live files (which are never compressed), for months/years
|
|
corresponding to found .tab files, are processed. Ticks from a
|
|
.tab-live file are only integrated if their timestamp is strictly later
|
|
than overallLastTabTickTime. Assumes such ticks are globally unique.
|
|
|
|
If a .tab-live file results in no new data being added under these rules,
|
|
it is deleted after processing.
|
|
|
|
The loading function automatically detects subsequent monthly/yearly files.
|
|
|
|
Main Function:
|
|
LoadTickDataSeries - Loads a series of tick data files.
|
|
------------------------------------------------------------------------------ *)
|
|
|
|
interface
|
|
|
|
uses
|
|
System.SysUtils,
|
|
System.Classes,
|
|
System.Generics.Collections,
|
|
System.IOUtils,
|
|
Myc.Futures,
|
|
Myc.Signals;
|
|
|
|
type
|
|
TDataSeries<T: record> = class
|
|
type
|
|
TDataPoint = packed record
|
|
OADateTime: Double;
|
|
Data: T;
|
|
end;
|
|
protected
|
|
class function ReadCompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
|
|
class function ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
|
|
public
|
|
class function LoadDataFile(var LoadGate: TLatch; const FileName: string): TFuture<TArray<TDataPoint>>; static;
|
|
class function LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint>>; static;
|
|
end;
|
|
|
|
TAskBidItem = packed record
|
|
Ask: Single;
|
|
Bid: Single;
|
|
end;
|
|
|
|
TAskBid = class(TDataSeries<TAskBidItem>)
|
|
type
|
|
TTick = TDataSeries<TAskBidItem>.TDataPoint;
|
|
end;
|
|
|
|
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue, MonthValue: Integer): Boolean;
|
|
|
|
// Finds the full filename of the oldest (=first) file for each symbol in the path.
|
|
function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
|
|
|
|
implementation
|
|
|
|
uses
|
|
System.Zip,
|
|
Myc.TaskManager;
|
|
|
|
// TryParseFileName parses filenames like "Symbol_Year_MM.Extension" or "Symbol_Year_MM.Extension_zip"
|
|
// to extract Path, Symbol, Year, and Month.
|
|
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue, MonthValue: Integer): Boolean;
|
|
var
|
|
fileNameNoPath: string;
|
|
nameForParsing: string;
|
|
parts: TArray<string>;
|
|
baseName: string;
|
|
begin
|
|
Result := False; // Default to failure
|
|
PathValue := ''; // Initialize all out parameters
|
|
SymbolValue := '';
|
|
YearValue := 0;
|
|
MonthValue := 0;
|
|
|
|
PathValue := TPath.GetDirectoryName(FileName);
|
|
fileNameNoPath := TPath.GetFileName(FileName);
|
|
|
|
nameForParsing := fileNameNoPath;
|
|
// Handle cases like "Symbol_Year_MM.Extension_zip" by removing the trailing "_zip" before extension removal
|
|
if nameForParsing.EndsWith('_zip', True) then
|
|
begin
|
|
// Get the part before "_zip"
|
|
baseName := nameForParsing.Substring(0, nameForParsing.Length - 4);
|
|
// Get the extension of the original name (e.g. ".csv_zip")
|
|
// and check if baseName still contains an extension that needs to be removed by GetFileNameWithoutExtension
|
|
if TPath.GetExtension(baseName) <> '' then
|
|
begin
|
|
nameForParsing := baseName;
|
|
end
|
|
else
|
|
begin
|
|
// If baseName (e.g. "EURUSD_2023_01") has no extension, it is the correct parsing base
|
|
// However, TPath.GetFileNameWithoutExtension would also work correctly here,
|
|
// but this handles a potential case where a file might be named "Symbol_Year_MM_zip" (no dot before zip)
|
|
end;
|
|
end;
|
|
|
|
// Remove the actual extension (e.g., ".csv", ".dat") from "Symbol_Year_MM.Extension" or the modified "Symbol_Year_MM.Extension" part of "_zip"
|
|
nameForParsing := TPath.GetFileNameWithoutExtension(nameForParsing); // e.g., "EURUSD_2023_01"
|
|
|
|
parts := nameForParsing.Split(['_']);
|
|
|
|
if Length(parts) < 3 then // Need at least Symbol_Year_Month
|
|
exit;
|
|
|
|
// Month is the last part.
|
|
if not TryStrToInt(parts[High(parts)], MonthValue) then
|
|
begin
|
|
exit;
|
|
end;
|
|
|
|
if (MonthValue < 1) or (MonthValue > 12) then // Validate month range
|
|
begin
|
|
MonthValue := 0; // Set to invalid if out of range
|
|
exit;
|
|
end;
|
|
|
|
// Year is the second to last part.
|
|
if not TryStrToInt(parts[High(parts) - 1], YearValue) then
|
|
begin
|
|
MonthValue := 0; // Also invalidate month if year parsing fails
|
|
exit;
|
|
end;
|
|
// Ensure YearValue is plausible, e.g. not 0 or negative, adjust as needed for your data's year range
|
|
if YearValue <= 0 then
|
|
begin
|
|
YearValue := 0;
|
|
MonthValue := 0;
|
|
exit;
|
|
end;
|
|
|
|
// Symbol consists of all parts before Year and Month.
|
|
SymbolValue := string.Join('_', Copy(parts, 0, Length(parts) - 2));
|
|
|
|
if SymbolValue = '' then // Ensure symbol is not empty
|
|
begin
|
|
// Reset all out parameters on failure before exit
|
|
PathValue := '';
|
|
YearValue := 0;
|
|
MonthValue := 0;
|
|
exit;
|
|
end;
|
|
|
|
Result := True;
|
|
end;
|
|
|
|
// Finds the oldest file for each symbol in the given directory.
|
|
// The file extension is ignored for the purpose of identifying the symbol, year, and month.
|
|
function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
|
|
type
|
|
TTrackedFileInfo = record
|
|
FullFileName: string;
|
|
Year: Integer;
|
|
Month: Integer;
|
|
end;
|
|
var
|
|
oldestFilesPerSymbol: TDictionary<string, TTrackedFileInfo>;
|
|
fileNames: TArray<string>;
|
|
currentFile: string;
|
|
parsedPath: string;
|
|
parsedSymbol: string;
|
|
parsedYear: Integer;
|
|
parsedMonth: Integer;
|
|
trackedInfo: TTrackedFileInfo;
|
|
begin
|
|
oldestFilesPerSymbol := TDictionary<string, TTrackedFileInfo>.Create; // Dictionary to track the oldest file per symbol
|
|
try
|
|
if not TDirectory.Exists(DirectoryPath) then
|
|
begin
|
|
// Or raise an exception, depending on desired error handling
|
|
exit;
|
|
end;
|
|
|
|
fileNames := TDirectory.GetFiles(DirectoryPath); // Get all files in the directory
|
|
|
|
for currentFile in fileNames do
|
|
begin
|
|
// Attempt to parse the file name
|
|
if TryParseFileName(currentFile, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
|
|
begin
|
|
// Check if this symbol is already tracked or if this file is older
|
|
if oldestFilesPerSymbol.TryGetValue(parsedSymbol, trackedInfo) then
|
|
begin
|
|
// If current file's year is less, or year is same and month is less, it's older
|
|
if (parsedYear < trackedInfo.Year) or ((parsedYear = trackedInfo.Year) and (parsedMonth < trackedInfo.Month)) then
|
|
begin
|
|
trackedInfo.FullFileName := currentFile; // Store full path to the file
|
|
trackedInfo.Year := parsedYear;
|
|
trackedInfo.Month := parsedMonth;
|
|
oldestFilesPerSymbol.AddOrSetValue(parsedSymbol, trackedInfo); // Update the entry for this symbol
|
|
end;
|
|
end
|
|
else
|
|
begin
|
|
// First time we see this symbol, so it's the oldest so far
|
|
trackedInfo.FullFileName := currentFile; // Store full path to the file
|
|
trackedInfo.Year := parsedYear;
|
|
trackedInfo.Month := parsedMonth;
|
|
oldestFilesPerSymbol.Add(parsedSymbol, trackedInfo); // Add new entry for this symbol
|
|
end;
|
|
end;
|
|
end;
|
|
|
|
// Populate the result array with the full names of the oldest files
|
|
SetLength(Result, oldestFilesPerSymbol.Count);
|
|
var n := 0;
|
|
for trackedInfo in oldestFilesPerSymbol.Values do
|
|
begin
|
|
Result[n] := trackedInfo.FullFileName;
|
|
inc(n);
|
|
end;
|
|
finally
|
|
oldestFilesPerSymbol.Free; // Free the dictionary
|
|
end;
|
|
end;
|
|
|
|
// Asynchronously reads tick data from a single specified file.
|
|
// Returns a TFuture that will provide an array of TAskBidSeries.
|
|
class function TDataSeries<T>.LoadDataFile(var LoadGate: TLatch; const FileName: string): TFuture<TArray<TDataPoint>>;
|
|
var
|
|
parsedPath, parsedSymbol: string;
|
|
parsedYear, parsedMonth: Integer;
|
|
begin
|
|
Result := TFuture<TArray<TDataPoint>>.Null;
|
|
|
|
// Attempt to parse year and month from the filename
|
|
if not TryParseFileName(FileName, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
|
|
exit;
|
|
|
|
if FileName.EndsWith('_zip', True) then
|
|
begin
|
|
var zipFileBytes := TFile.ReadAllBytes(FileName); // Read all bytes of the .zip file
|
|
if Length(zipFileBytes) = 0 then
|
|
exit;
|
|
|
|
var capFileName := FileName;
|
|
Result :=
|
|
TFuture<TBytes>
|
|
.Construct(
|
|
TLatch.Enqueue(LoadGate),
|
|
function: TBytes
|
|
begin
|
|
Result := TFile.ReadAllBytes(capFileName); // Read all bytes of the .zip file
|
|
end)
|
|
.Chain<TArray<TDataPoint>>(
|
|
function(bytes: TBytes): TArray<TDataPoint>
|
|
begin
|
|
var zipMemoryStream := TBytesStream.Create(zipFileBytes); // Create a memory stream to hold the zip file content
|
|
try
|
|
zipMemoryStream.Position := 0;
|
|
Result := ReadCompressedData(zipMemoryStream);
|
|
finally
|
|
zipMemoryStream.Free;
|
|
end;
|
|
end);
|
|
end
|
|
else if TFile.Exists(FileName) then
|
|
begin
|
|
var capFileName := FileName;
|
|
Result :=
|
|
TFuture<TArray<TDataPoint>>.Construct(
|
|
function: TArray<TDataPoint>
|
|
begin
|
|
if TFile.Exists(capFileName) then
|
|
begin
|
|
var stream := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
|
|
try
|
|
try
|
|
Result := ReadUncompressedData(stream);
|
|
except
|
|
on E: EReadError do
|
|
begin
|
|
raise EReadError.CreateFmt('File %s: %s', [capFileName, E.Message]);
|
|
end;
|
|
end;
|
|
finally
|
|
stream.Free;
|
|
end;
|
|
end;
|
|
end
|
|
);
|
|
end;
|
|
end;
|
|
|
|
class function TDataSeries<T>.LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint>>;
|
|
// Local type declaration for storing file information for phase 1 processing
|
|
type
|
|
TPhase1FileEntry = record
|
|
Path: string;
|
|
Year: Integer;
|
|
Month: Integer;
|
|
end;
|
|
var
|
|
initialYear, initialMonth: Integer;
|
|
currentTabYear, currentTabMonth: Integer;
|
|
pathName, symbolName: string;
|
|
tabFileEntryRecord: TPhase1FileEntry; // Variable to construct record instances
|
|
loadedState: TState;
|
|
loadedFiles: TArray<TFuture<TArray<TDataPoint>>>;
|
|
liveData: TFuture<TArray<TDataPoint>>;
|
|
|
|
begin // Start of LoadDataSeries
|
|
if not TryParseFileName(FileName, pathName, symbolName, initialYear, initialMonth) then
|
|
begin
|
|
raise EArgumentException.CreateFmt(
|
|
'Invalid initial file name format: %s. Expected Symbol_Year_MM.Extension or Symbol_Year_MM.Extension_zip',
|
|
[FileName]);
|
|
end;
|
|
|
|
currentTabYear := initialYear;
|
|
currentTabMonth := initialMonth;
|
|
var tabBaseNameFormat: string;
|
|
var compressedTabPath, uncompressedTabPath: string;
|
|
var actualTabFileToLoad: string;
|
|
var maxTabYearFound := -1;
|
|
var maxTabMonthFound := -1;
|
|
|
|
var filesToLoadPhase1 := TList<TPhase1FileEntry>.Create; // Create list with the named record type
|
|
try
|
|
while True do
|
|
begin
|
|
tabBaseNameFormat := Format('%s_%.4d_%.2d.tab', [symbolName, currentTabYear, currentTabMonth]);
|
|
compressedTabPath := TPath.Combine(pathName, tabBaseNameFormat + '_zip');
|
|
uncompressedTabPath := TPath.Combine(pathName, tabBaseNameFormat);
|
|
|
|
actualTabFileToLoad := '';
|
|
if TFile.Exists(compressedTabPath) then
|
|
begin
|
|
actualTabFileToLoad := compressedTabPath;
|
|
end
|
|
else if TFile.Exists(uncompressedTabPath) then
|
|
begin
|
|
actualTabFileToLoad := uncompressedTabPath;
|
|
end;
|
|
|
|
if actualTabFileToLoad = '' then
|
|
break; // No more sequential .tab files
|
|
|
|
// Construct the record and add it to the list
|
|
tabFileEntryRecord.Path := actualTabFileToLoad;
|
|
tabFileEntryRecord.Year := currentTabYear;
|
|
tabFileEntryRecord.Month := currentTabMonth;
|
|
filesToLoadPhase1.Add(tabFileEntryRecord);
|
|
|
|
// Advance to the next month
|
|
Inc(currentTabMonth);
|
|
if currentTabMonth > 12 then
|
|
begin
|
|
currentTabMonth := 1;
|
|
Inc(currentTabYear);
|
|
end;
|
|
end;
|
|
|
|
// Now load and process identified .tab files
|
|
var loadedFileList := TList<TFuture<TArray<TDataPoint>>>.Create;
|
|
var loadStates := TList<TState>.Create;
|
|
try
|
|
var LoadGate: TLatch;
|
|
|
|
for var tabFileEntry: TPhase1FileEntry in filesToLoadPhase1 do // Iterate with the correct type
|
|
begin
|
|
var data := LoadDataFile(LoadGate, tabFileEntry.Path);
|
|
|
|
loadedFileList.Add(TFuture<TArray<TDataPoint>>.Create(data));
|
|
loadStates.Add(data.Done);
|
|
|
|
if tabFileEntry.Year > maxTabYearFound then
|
|
begin
|
|
maxTabYearFound := tabFileEntry.Year;
|
|
maxTabMonthFound := tabFileEntry.Month;
|
|
end
|
|
else if (tabFileEntry.Year = maxTabYearFound) and (tabFileEntry.Month > maxTabMonthFound) then
|
|
begin
|
|
maxTabMonthFound := tabFileEntry.Month;
|
|
end;
|
|
end;
|
|
|
|
var liveFileBaseName := Format('%s_%d_%02d.tab-live', [symbolName, maxTabYearFound, maxTabMonthFound]);
|
|
liveData := LoadDataFile(LoadGate, TPath.Combine(pathName, liveFileBaseName));
|
|
loadStates.Add(liveData.Done);
|
|
loadedFileList.Add(liveData);
|
|
|
|
loadedState := TState.All(loadStates.ToArray);
|
|
loadedFiles := loadedFileList.ToArray;
|
|
finally
|
|
loadStates.Free;
|
|
loadedFileList.Free;
|
|
end;
|
|
finally
|
|
filesToLoadPhase1.Free;
|
|
end;
|
|
|
|
Result :=
|
|
TFuture<TArray<TDataPoint>>.Construct(
|
|
loadedState,
|
|
function: TArray<TDataPoint>
|
|
begin
|
|
var tickList := TList<TDataPoint>.Create;
|
|
try
|
|
var totalTicks := Length(liveData.Result);
|
|
var overallLastTabTickTime := 0.0;
|
|
|
|
for var P in loadedFiles do
|
|
Inc(totalTicks, Length(P.Result));
|
|
|
|
tickList.Capacity := totalTicks;
|
|
|
|
for var P in loadedFiles do
|
|
begin
|
|
if Length(P.Result) > 0 then
|
|
begin
|
|
for var iterTickData in P.Result do
|
|
if iterTickData.OADateTime > overallLastTabTickTime then
|
|
begin
|
|
tickList.Add(iterTickData);
|
|
overallLastTabTickTime := iterTickData.OADateTime;
|
|
end;
|
|
end;
|
|
end;
|
|
|
|
Result := tickList.ToArray;
|
|
finally
|
|
tickList.Free;
|
|
end;
|
|
end
|
|
);
|
|
end;
|
|
|
|
class function TDataSeries<T>.ReadCompressedData(const InputStream: TStream): TArray<TDataPoint>;
|
|
var
|
|
decompressionStream: TStream; // Holds uncompressed data of a single zip entry
|
|
localHeader: TZipHeader;
|
|
entryIndex: Integer;
|
|
i: Integer;
|
|
zipFileInstance: TZipFile;
|
|
begin
|
|
// Initialize Result record
|
|
SetLength(Result, 0);
|
|
|
|
decompressionStream := nil;
|
|
zipFileInstance := nil;
|
|
|
|
try
|
|
InputStream.Position := 0; // Reset stream position to the beginning for TZipFile
|
|
|
|
// Open TZipFile from this memory stream
|
|
zipFileInstance := TZipFile.Create;
|
|
zipFileInstance.Open(InputStream, TZipMode.zmRead); // Open the zip from the memory stream
|
|
|
|
if zipFileInstance.FileCount = 0 then
|
|
begin
|
|
// If TZipFile opens an empty or invalid stream successfully but finds no files
|
|
exit; // FileSuccessfullyLoaded remains False
|
|
end;
|
|
|
|
entryIndex := -1;
|
|
for i := 0 to zipFileInstance.FileCount - 1 do
|
|
begin
|
|
if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
|
|
begin
|
|
entryIndex := i;
|
|
break;
|
|
end;
|
|
end;
|
|
|
|
if entryIndex = -1 then // If specific name not found, default to the first entry
|
|
begin
|
|
if zipFileInstance.FileCount > 0 then
|
|
entryIndex := 0
|
|
else
|
|
exit; // Should not happen if FileCount > 0 check passed
|
|
end;
|
|
|
|
// Decompress the specific entry from the in-memory zip file.
|
|
// TZipFile.Read creates and populates decompressionStream with uncompressed data.
|
|
zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
|
|
|
|
if not Assigned(decompressionStream) then
|
|
exit; // Exit if no valid stream could be prepared
|
|
|
|
Result := ReadUncompressedData(decompressionStream);
|
|
finally
|
|
// Ensure all potentially created objects are freed
|
|
if Assigned(decompressionStream) then // Holds uncompressed data from a zip entry
|
|
decompressionStream.Free;
|
|
if Assigned(zipFileInstance) then
|
|
zipFileInstance.Free;
|
|
end;
|
|
end;
|
|
|
|
class function TDataSeries<T>.ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint>;
|
|
var
|
|
fileSize: Int64;
|
|
recordCount: Integer;
|
|
bytesRead: Integer;
|
|
begin
|
|
// Initialize Result record
|
|
SetLength(Result, 0);
|
|
|
|
InputStream.Position := 0;
|
|
|
|
fileSize := InputStream.Size;
|
|
if fileSize = 0 then
|
|
exit;
|
|
|
|
if (fileSize mod SizeOf(TDataPoint)) <> 0 then
|
|
raise EReadError.CreateFmt('File corrupted. Size %d is not a multiple of record size %d.', [fileSize, SizeOf(TDataPoint)]);
|
|
|
|
recordCount := fileSize div SizeOf(TDataPoint);
|
|
if recordCount > 0 then
|
|
begin
|
|
SetLength(Result, recordCount);
|
|
bytesRead := InputStream.Read(Result[0], fileSize);
|
|
if bytesRead <> fileSize then
|
|
raise EReadError.CreateFmt('Read error. Expected to read %d bytes, but actually read %d bytes.', [fileSize, bytesRead]);
|
|
end;
|
|
end;
|
|
|
|
end.
|