TDataSeries<T>
This commit is contained in:
Michael Schimmel
@@ -0,0 +1,542 @@
|
||||
unit Myc.Trade.DataSeries;
|
||||
|
||||
(* ------------------------------------------------------------------------------
|
||||
This unit provides functions to load tick data (TTickData) from
|
||||
binary files. The data originates from a C# application
|
||||
and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
|
||||
|
||||
The files follow the naming convention:
|
||||
For .tab files: Symbol_Year_MM.tab OR Symbol_Year_MM.tab_zip
|
||||
For .tab-live files: Symbol_Year_MM.tab-live (NEVER compressed)
|
||||
(MM represents the two-digit month, e.g., 01 for January, 12 for December)
|
||||
|
||||
If both compressed (.tab_zip) and uncompressed (.tab) versions of a .tab
|
||||
file exist, the compressed version is preferred. Files ending with _zip
|
||||
are expected to be ZIP archives containing the actual data file
|
||||
(e.g., Symbol_Year_MM.tab).
|
||||
|
||||
Loading occurs in two phases:
|
||||
1. All .tab files (regular or _zip) are processed chronologically by
|
||||
month and year. Unique ticks are loaded, and the timestamp of the
|
||||
latest tick across all .tab files is determined (overallLastTabTickTime).
|
||||
Assumes timestamps are unique across all .tab files.
|
||||
2. All .tab-live files (which are never compressed), for months/years
|
||||
corresponding to found .tab files, are processed. Ticks from a
|
||||
.tab-live file are only integrated if their timestamp is strictly later
|
||||
than overallLastTabTickTime. Assumes such ticks are globally unique.
|
||||
|
||||
If a .tab-live file results in no new data being added under these rules,
|
||||
it is deleted after processing.
|
||||
|
||||
The loading function automatically detects subsequent monthly/yearly files.
|
||||
|
||||
Main Function:
|
||||
LoadTickDataSeries - Loads a series of tick data files.
|
||||
------------------------------------------------------------------------------ *)
|
||||
|
||||
interface
|
||||
|
||||
uses
|
||||
System.SysUtils,
|
||||
System.Classes,
|
||||
System.Generics.Collections,
|
||||
System.IOUtils,
|
||||
Myc.Futures,
|
||||
Myc.Signals;
|
||||
|
||||
type
|
||||
TDataSeries<T: record> = class
|
||||
type
|
||||
TDataPoint = packed record
|
||||
OADateTime: Double;
|
||||
Data: T;
|
||||
end;
|
||||
protected
|
||||
class function ReadCompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
|
||||
class function ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
|
||||
public
|
||||
class function LoadDataFile(var LoadGate: TLatch; const FileName: string): TFuture<TArray<TDataPoint>>; static;
|
||||
class function LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint>>; static;
|
||||
end;
|
||||
|
||||
TAskBidItem = packed record
|
||||
Ask: Single;
|
||||
Bid: Single;
|
||||
end;
|
||||
|
||||
TAskBid = class(TDataSeries<TAskBidItem>)
|
||||
type
|
||||
TTick = TDataSeries<TAskBidItem>.TDataPoint;
|
||||
end;
|
||||
|
||||
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue, MonthValue: Integer): Boolean;
|
||||
|
||||
// Finds the full filename of the oldest (=first) file for each symbol in the path.
|
||||
function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
|
||||
|
||||
implementation
|
||||
|
||||
uses
|
||||
System.Zip,
|
||||
Myc.TaskManager;
|
||||
|
||||
// TryParseFileName parses filenames like "Symbol_Year_MM.Extension" or "Symbol_Year_MM.Extension_zip"
|
||||
// to extract Path, Symbol, Year, and Month.
|
||||
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue, MonthValue: Integer): Boolean;
|
||||
var
|
||||
fileNameNoPath: string;
|
||||
nameForParsing: string;
|
||||
parts: TArray<string>;
|
||||
baseName: string;
|
||||
begin
|
||||
Result := False; // Default to failure
|
||||
PathValue := ''; // Initialize all out parameters
|
||||
SymbolValue := '';
|
||||
YearValue := 0;
|
||||
MonthValue := 0;
|
||||
|
||||
PathValue := TPath.GetDirectoryName(FileName);
|
||||
fileNameNoPath := TPath.GetFileName(FileName);
|
||||
|
||||
nameForParsing := fileNameNoPath;
|
||||
// Handle cases like "Symbol_Year_MM.Extension_zip" by removing the trailing "_zip" before extension removal
|
||||
if nameForParsing.EndsWith('_zip', True) then
|
||||
begin
|
||||
// Get the part before "_zip"
|
||||
baseName := nameForParsing.Substring(0, nameForParsing.Length - 4);
|
||||
// Get the extension of the original name (e.g. ".csv_zip")
|
||||
// and check if baseName still contains an extension that needs to be removed by GetFileNameWithoutExtension
|
||||
if TPath.GetExtension(baseName) <> '' then
|
||||
begin
|
||||
nameForParsing := baseName;
|
||||
end
|
||||
else
|
||||
begin
|
||||
// If baseName (e.g. "EURUSD_2023_01") has no extension, it is the correct parsing base
|
||||
// However, TPath.GetFileNameWithoutExtension would also work correctly here,
|
||||
// but this handles a potential case where a file might be named "Symbol_Year_MM_zip" (no dot before zip)
|
||||
end;
|
||||
end;
|
||||
|
||||
// Remove the actual extension (e.g., ".csv", ".dat") from "Symbol_Year_MM.Extension" or the modified "Symbol_Year_MM.Extension" part of "_zip"
|
||||
nameForParsing := TPath.GetFileNameWithoutExtension(nameForParsing); // e.g., "EURUSD_2023_01"
|
||||
|
||||
parts := nameForParsing.Split(['_']);
|
||||
|
||||
if Length(parts) < 3 then // Need at least Symbol_Year_Month
|
||||
exit;
|
||||
|
||||
// Month is the last part.
|
||||
if not TryStrToInt(parts[High(parts)], MonthValue) then
|
||||
begin
|
||||
exit;
|
||||
end;
|
||||
|
||||
if (MonthValue < 1) or (MonthValue > 12) then // Validate month range
|
||||
begin
|
||||
MonthValue := 0; // Set to invalid if out of range
|
||||
exit;
|
||||
end;
|
||||
|
||||
// Year is the second to last part.
|
||||
if not TryStrToInt(parts[High(parts) - 1], YearValue) then
|
||||
begin
|
||||
MonthValue := 0; // Also invalidate month if year parsing fails
|
||||
exit;
|
||||
end;
|
||||
// Ensure YearValue is plausible, e.g. not 0 or negative, adjust as needed for your data's year range
|
||||
if YearValue <= 0 then
|
||||
begin
|
||||
YearValue := 0;
|
||||
MonthValue := 0;
|
||||
exit;
|
||||
end;
|
||||
|
||||
// Symbol consists of all parts before Year and Month.
|
||||
SymbolValue := string.Join('_', Copy(parts, 0, Length(parts) - 2));
|
||||
|
||||
if SymbolValue = '' then // Ensure symbol is not empty
|
||||
begin
|
||||
// Reset all out parameters on failure before exit
|
||||
PathValue := '';
|
||||
YearValue := 0;
|
||||
MonthValue := 0;
|
||||
exit;
|
||||
end;
|
||||
|
||||
Result := True;
|
||||
end;
|
||||
|
||||
// Finds the oldest file for each symbol in the given directory.
|
||||
// The file extension is ignored for the purpose of identifying the symbol, year, and month.
|
||||
function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
|
||||
type
|
||||
TTrackedFileInfo = record
|
||||
FullFileName: string;
|
||||
Year: Integer;
|
||||
Month: Integer;
|
||||
end;
|
||||
var
|
||||
oldestFilesPerSymbol: TDictionary<string, TTrackedFileInfo>;
|
||||
fileNames: TArray<string>;
|
||||
currentFile: string;
|
||||
parsedPath: string;
|
||||
parsedSymbol: string;
|
||||
parsedYear: Integer;
|
||||
parsedMonth: Integer;
|
||||
trackedInfo: TTrackedFileInfo;
|
||||
begin
|
||||
oldestFilesPerSymbol := TDictionary<string, TTrackedFileInfo>.Create; // Dictionary to track the oldest file per symbol
|
||||
try
|
||||
if not TDirectory.Exists(DirectoryPath) then
|
||||
begin
|
||||
// Or raise an exception, depending on desired error handling
|
||||
exit;
|
||||
end;
|
||||
|
||||
fileNames := TDirectory.GetFiles(DirectoryPath); // Get all files in the directory
|
||||
|
||||
for currentFile in fileNames do
|
||||
begin
|
||||
// Attempt to parse the file name
|
||||
if TryParseFileName(currentFile, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
|
||||
begin
|
||||
// Check if this symbol is already tracked or if this file is older
|
||||
if oldestFilesPerSymbol.TryGetValue(parsedSymbol, trackedInfo) then
|
||||
begin
|
||||
// If current file's year is less, or year is same and month is less, it's older
|
||||
if (parsedYear < trackedInfo.Year) or ((parsedYear = trackedInfo.Year) and (parsedMonth < trackedInfo.Month)) then
|
||||
begin
|
||||
trackedInfo.FullFileName := currentFile; // Store full path to the file
|
||||
trackedInfo.Year := parsedYear;
|
||||
trackedInfo.Month := parsedMonth;
|
||||
oldestFilesPerSymbol.AddOrSetValue(parsedSymbol, trackedInfo); // Update the entry for this symbol
|
||||
end;
|
||||
end
|
||||
else
|
||||
begin
|
||||
// First time we see this symbol, so it's the oldest so far
|
||||
trackedInfo.FullFileName := currentFile; // Store full path to the file
|
||||
trackedInfo.Year := parsedYear;
|
||||
trackedInfo.Month := parsedMonth;
|
||||
oldestFilesPerSymbol.Add(parsedSymbol, trackedInfo); // Add new entry for this symbol
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
// Populate the result array with the full names of the oldest files
|
||||
SetLength(Result, oldestFilesPerSymbol.Count);
|
||||
var n := 0;
|
||||
for trackedInfo in oldestFilesPerSymbol.Values do
|
||||
begin
|
||||
Result[n] := trackedInfo.FullFileName;
|
||||
inc(n);
|
||||
end;
|
||||
finally
|
||||
oldestFilesPerSymbol.Free; // Free the dictionary
|
||||
end;
|
||||
end;
|
||||
|
||||
// Asynchronously reads tick data from a single specified file.
|
||||
// Returns a TFuture that will provide an array of TAskBidSeries.
|
||||
class function TDataSeries<T>.LoadDataFile(var LoadGate: TLatch; const FileName: string): TFuture<TArray<TDataPoint>>;
|
||||
var
|
||||
parsedPath, parsedSymbol: string;
|
||||
parsedYear, parsedMonth: Integer;
|
||||
begin
|
||||
Result := TFuture<TArray<TDataPoint>>.Null;
|
||||
|
||||
// Attempt to parse year and month from the filename
|
||||
if not TryParseFileName(FileName, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
|
||||
exit;
|
||||
|
||||
if FileName.EndsWith('_zip', True) then
|
||||
begin
|
||||
var zipFileBytes := TFile.ReadAllBytes(FileName); // Read all bytes of the .zip file
|
||||
if Length(zipFileBytes) = 0 then
|
||||
exit;
|
||||
|
||||
var capFileName := FileName;
|
||||
Result :=
|
||||
TFuture<TBytes>
|
||||
.Construct(
|
||||
TLatch.Enqueue(LoadGate),
|
||||
function: TBytes
|
||||
begin
|
||||
Result := TFile.ReadAllBytes(capFileName); // Read all bytes of the .zip file
|
||||
end)
|
||||
.Chain<TArray<TDataPoint>>(
|
||||
function(bytes: TBytes): TArray<TDataPoint>
|
||||
begin
|
||||
var zipMemoryStream := TBytesStream.Create(zipFileBytes); // Create a memory stream to hold the zip file content
|
||||
try
|
||||
zipMemoryStream.Position := 0;
|
||||
Result := ReadCompressedData(zipMemoryStream);
|
||||
finally
|
||||
zipMemoryStream.Free;
|
||||
end;
|
||||
end);
|
||||
end
|
||||
else if TFile.Exists(FileName) then
|
||||
begin
|
||||
var capFileName := FileName;
|
||||
Result :=
|
||||
TFuture<TArray<TDataPoint>>.Construct(
|
||||
function: TArray<TDataPoint>
|
||||
begin
|
||||
if TFile.Exists(capFileName) then
|
||||
begin
|
||||
var stream := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
|
||||
try
|
||||
try
|
||||
Result := ReadUncompressedData(stream);
|
||||
except
|
||||
on E: EReadError do
|
||||
begin
|
||||
raise EReadError.CreateFmt('File %s: %s', [capFileName, E.Message]);
|
||||
end;
|
||||
end;
|
||||
finally
|
||||
stream.Free;
|
||||
end;
|
||||
end;
|
||||
end
|
||||
);
|
||||
end;
|
||||
end;
|
||||
|
||||
class function TDataSeries<T>.LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint>>;
|
||||
// Local type declaration for storing file information for phase 1 processing
|
||||
type
|
||||
TPhase1FileEntry = record
|
||||
Path: string;
|
||||
Year: Integer;
|
||||
Month: Integer;
|
||||
end;
|
||||
var
|
||||
initialYear, initialMonth: Integer;
|
||||
currentTabYear, currentTabMonth: Integer;
|
||||
pathName, symbolName: string;
|
||||
tabFileEntryRecord: TPhase1FileEntry; // Variable to construct record instances
|
||||
loadedState: TState;
|
||||
loadedFiles: TArray<TFuture<TArray<TDataPoint>>>;
|
||||
liveData: TFuture<TArray<TDataPoint>>;
|
||||
|
||||
begin // Start of LoadDataSeries
|
||||
if not TryParseFileName(FileName, pathName, symbolName, initialYear, initialMonth) then
|
||||
begin
|
||||
raise EArgumentException.CreateFmt(
|
||||
'Invalid initial file name format: %s. Expected Symbol_Year_MM.Extension or Symbol_Year_MM.Extension_zip',
|
||||
[FileName]);
|
||||
end;
|
||||
|
||||
currentTabYear := initialYear;
|
||||
currentTabMonth := initialMonth;
|
||||
var tabBaseNameFormat: string;
|
||||
var compressedTabPath, uncompressedTabPath: string;
|
||||
var actualTabFileToLoad: string;
|
||||
var maxTabYearFound := -1;
|
||||
var maxTabMonthFound := -1;
|
||||
|
||||
var filesToLoadPhase1 := TList<TPhase1FileEntry>.Create; // Create list with the named record type
|
||||
try
|
||||
while True do
|
||||
begin
|
||||
tabBaseNameFormat := Format('%s_%.4d_%.2d.tab', [symbolName, currentTabYear, currentTabMonth]);
|
||||
compressedTabPath := TPath.Combine(pathName, tabBaseNameFormat + '_zip');
|
||||
uncompressedTabPath := TPath.Combine(pathName, tabBaseNameFormat);
|
||||
|
||||
actualTabFileToLoad := '';
|
||||
if TFile.Exists(compressedTabPath) then
|
||||
begin
|
||||
actualTabFileToLoad := compressedTabPath;
|
||||
end
|
||||
else if TFile.Exists(uncompressedTabPath) then
|
||||
begin
|
||||
actualTabFileToLoad := uncompressedTabPath;
|
||||
end;
|
||||
|
||||
if actualTabFileToLoad = '' then
|
||||
break; // No more sequential .tab files
|
||||
|
||||
// Construct the record and add it to the list
|
||||
tabFileEntryRecord.Path := actualTabFileToLoad;
|
||||
tabFileEntryRecord.Year := currentTabYear;
|
||||
tabFileEntryRecord.Month := currentTabMonth;
|
||||
filesToLoadPhase1.Add(tabFileEntryRecord);
|
||||
|
||||
// Advance to the next month
|
||||
Inc(currentTabMonth);
|
||||
if currentTabMonth > 12 then
|
||||
begin
|
||||
currentTabMonth := 1;
|
||||
Inc(currentTabYear);
|
||||
end;
|
||||
end;
|
||||
|
||||
// Now load and process identified .tab files
|
||||
var loadedFileList := TList<TFuture<TArray<TDataPoint>>>.Create;
|
||||
var loadStates := TList<TState>.Create;
|
||||
try
|
||||
var LoadGate: TLatch;
|
||||
|
||||
for var tabFileEntry: TPhase1FileEntry in filesToLoadPhase1 do // Iterate with the correct type
|
||||
begin
|
||||
var data := LoadDataFile(LoadGate, tabFileEntry.Path);
|
||||
|
||||
loadedFileList.Add(TFuture<TArray<TDataPoint>>.Create(data));
|
||||
loadStates.Add(data.Done);
|
||||
|
||||
if tabFileEntry.Year > maxTabYearFound then
|
||||
begin
|
||||
maxTabYearFound := tabFileEntry.Year;
|
||||
maxTabMonthFound := tabFileEntry.Month;
|
||||
end
|
||||
else if (tabFileEntry.Year = maxTabYearFound) and (tabFileEntry.Month > maxTabMonthFound) then
|
||||
begin
|
||||
maxTabMonthFound := tabFileEntry.Month;
|
||||
end;
|
||||
end;
|
||||
|
||||
var liveFileBaseName := Format('%s_%d_%02d.tab-live', [symbolName, maxTabYearFound, maxTabMonthFound]);
|
||||
liveData := LoadDataFile(LoadGate, TPath.Combine(pathName, liveFileBaseName));
|
||||
loadStates.Add(liveData.Done);
|
||||
loadedFileList.Add(liveData);
|
||||
|
||||
loadedState := TState.All(loadStates.ToArray);
|
||||
loadedFiles := loadedFileList.ToArray;
|
||||
finally
|
||||
loadStates.Free;
|
||||
loadedFileList.Free;
|
||||
end;
|
||||
finally
|
||||
filesToLoadPhase1.Free;
|
||||
end;
|
||||
|
||||
Result :=
|
||||
TFuture<TArray<TDataPoint>>.Construct(
|
||||
loadedState,
|
||||
function: TArray<TDataPoint>
|
||||
begin
|
||||
var tickList := TList<TDataPoint>.Create;
|
||||
try
|
||||
var totalTicks := Length(liveData.Result);
|
||||
var overallLastTabTickTime := 0.0;
|
||||
|
||||
for var P in loadedFiles do
|
||||
Inc(totalTicks, Length(P.Result));
|
||||
|
||||
tickList.Capacity := totalTicks;
|
||||
|
||||
for var P in loadedFiles do
|
||||
begin
|
||||
if Length(P.Result) > 0 then
|
||||
begin
|
||||
for var iterTickData in P.Result do
|
||||
if iterTickData.OADateTime > overallLastTabTickTime then
|
||||
begin
|
||||
tickList.Add(iterTickData);
|
||||
overallLastTabTickTime := iterTickData.OADateTime;
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
Result := tickList.ToArray;
|
||||
finally
|
||||
tickList.Free;
|
||||
end;
|
||||
end
|
||||
);
|
||||
end;
|
||||
|
||||
class function TDataSeries<T>.ReadCompressedData(const InputStream: TStream): TArray<TDataPoint>;
|
||||
var
|
||||
decompressionStream: TStream; // Holds uncompressed data of a single zip entry
|
||||
localHeader: TZipHeader;
|
||||
entryIndex: Integer;
|
||||
i: Integer;
|
||||
zipFileInstance: TZipFile;
|
||||
begin
|
||||
// Initialize Result record
|
||||
SetLength(Result, 0);
|
||||
|
||||
decompressionStream := nil;
|
||||
zipFileInstance := nil;
|
||||
|
||||
try
|
||||
InputStream.Position := 0; // Reset stream position to the beginning for TZipFile
|
||||
|
||||
// Open TZipFile from this memory stream
|
||||
zipFileInstance := TZipFile.Create;
|
||||
zipFileInstance.Open(InputStream, TZipMode.zmRead); // Open the zip from the memory stream
|
||||
|
||||
if zipFileInstance.FileCount = 0 then
|
||||
begin
|
||||
// If TZipFile opens an empty or invalid stream successfully but finds no files
|
||||
exit; // FileSuccessfullyLoaded remains False
|
||||
end;
|
||||
|
||||
entryIndex := -1;
|
||||
for i := 0 to zipFileInstance.FileCount - 1 do
|
||||
begin
|
||||
if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
|
||||
begin
|
||||
entryIndex := i;
|
||||
break;
|
||||
end;
|
||||
end;
|
||||
|
||||
if entryIndex = -1 then // If specific name not found, default to the first entry
|
||||
begin
|
||||
if zipFileInstance.FileCount > 0 then
|
||||
entryIndex := 0
|
||||
else
|
||||
exit; // Should not happen if FileCount > 0 check passed
|
||||
end;
|
||||
|
||||
// Decompress the specific entry from the in-memory zip file.
|
||||
// TZipFile.Read creates and populates decompressionStream with uncompressed data.
|
||||
zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
|
||||
|
||||
if not Assigned(decompressionStream) then
|
||||
exit; // Exit if no valid stream could be prepared
|
||||
|
||||
Result := ReadUncompressedData(decompressionStream);
|
||||
finally
|
||||
// Ensure all potentially created objects are freed
|
||||
if Assigned(decompressionStream) then // Holds uncompressed data from a zip entry
|
||||
decompressionStream.Free;
|
||||
if Assigned(zipFileInstance) then
|
||||
zipFileInstance.Free;
|
||||
end;
|
||||
end;
|
||||
|
||||
class function TDataSeries<T>.ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint>;
|
||||
var
|
||||
fileSize: Int64;
|
||||
recordCount: Integer;
|
||||
bytesRead: Integer;
|
||||
begin
|
||||
// Initialize Result record
|
||||
SetLength(Result, 0);
|
||||
|
||||
InputStream.Position := 0;
|
||||
|
||||
fileSize := InputStream.Size;
|
||||
if fileSize = 0 then
|
||||
exit;
|
||||
|
||||
if (fileSize mod SizeOf(TDataPoint)) <> 0 then
|
||||
raise EReadError.CreateFmt('File corrupted. Size %d is not a multiple of record size %d.', [fileSize, SizeOf(TDataPoint)]);
|
||||
|
||||
recordCount := fileSize div SizeOf(TDataPoint);
|
||||
if recordCount > 0 then
|
||||
begin
|
||||
SetLength(Result, recordCount);
|
||||
bytesRead := InputStream.Read(Result[0], fileSize);
|
||||
if bytesRead <> fileSize then
|
||||
raise EReadError.CreateFmt('Read error. Expected to read %d bytes, but actually read %d bytes.', [fileSize, bytesRead]);
|
||||
end;
|
||||
end;
|
||||
|
||||
end.
|
||||
Reference in New Issue
Block a user