Michael Schimmel
255 lines
10 KiB
ObjectPascal
255 lines
10 KiB
ObjectPascal
unit Myc.TickLoader;
|
|
|
|
(*------------------------------------------------------------------------------
|
|
Unit: Myc.TickLoader
|
|
Author: Delphi Algo Trading (Assistant)
|
|
Date: May 22, 2025
|
|
Description:
|
|
This unit provides functions to load tick data (TTickData) from
|
|
binary files. The data originates from a C# application
|
|
and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
|
|
|
|
The files now follow the naming convention: Symbol_Year.tab.
|
|
(Prefix has been removed from the convention).
|
|
Loading occurs in two phases:
|
|
1. All .tab files are processed chronologically. Unique ticks are loaded,
|
|
and the timestamp of the latest tick across all .tab files is determined
|
|
(overallLastTabTickTime). Assumes timestamps are unique across all .tab files.
|
|
2. All .tab-live files (for years corresponding to found .tab files) are
|
|
processed. Ticks from a .tab-live file are only integrated if their
|
|
timestamp is strictly later than overallLastTabTickTime. Assumes such
|
|
ticks are globally unique.
|
|
|
|
If a .tab-live file results in no new data being added under these rules,
|
|
it is deleted after processing.
|
|
|
|
The loading function automatically detects subsequent yearly .tab files.
|
|
|
|
Main Function:
|
|
LoadTickDataSeries - Loads a series of tick data files.
|
|
------------------------------------------------------------------------------*)
|
|
|
|
interface
|
|
|
|
uses
|
|
System.SysUtils,
|
|
System.Classes,
|
|
System.Generics.Collections,
|
|
System.IOUtils;
|
|
|
|
type
|
|
TTickData = packed record
|
|
OADateTime: Double;
|
|
Ask: Single;
|
|
Bid: Single;
|
|
end;
|
|
PTTickData = ^TTickData;
|
|
|
|
// TryParseFileName now expects filenames like "Symbol_Year" (or "Symbol_Part2_Year")
|
|
// PrefixValue will be returned as an empty string.
|
|
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue: Integer): Boolean;
|
|
function LoadTickDataSeries(const FileName: string): TArray<TTickData>;
|
|
|
|
implementation
|
|
|
|
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue: Integer): Boolean;
|
|
var
|
|
fileNameOnly: string;
|
|
parts: TArray<string>;
|
|
begin
|
|
Result := False;
|
|
PathValue := TPath.GetDirectoryName(FileName);
|
|
fileNameOnly := TPath.GetFileNameWithoutExtension(FileName); // Handles any extension for parsing basename
|
|
|
|
parts := fileNameOnly.Split(['_']);
|
|
|
|
// Expected format is Symbol_Year or SymbolPart1_SymbolPart2_Year, so at least 2 parts.
|
|
if Length(parts) < 2 then
|
|
begin
|
|
SymbolValue := ''; YearValue := 0; // Clear out params on minimum parts failure
|
|
Exit;
|
|
end;
|
|
|
|
// Year is the last part.
|
|
if not TryStrToInt(parts[High(parts)], YearValue) then
|
|
begin
|
|
SymbolValue := ''; // Clear SymbolValue as well on year parse failure
|
|
Exit;
|
|
end;
|
|
|
|
// Symbol consists of all parts before the year part.
|
|
// Copy all parts except the last one (which is the Year).
|
|
// The second parameter of Copy (for TArray<string>) is the starting index,
|
|
// the third parameter is the count of elements to copy.
|
|
// We want to copy elements from index 0 up to (but not including) the last element.
|
|
// So, the count is High(parts) (which is Length(parts) - 1).
|
|
if High(parts) > 0 then // Ensure there's at least one part for the symbol
|
|
SymbolValue := string.Join('_', Copy(parts, 0, High(parts)))
|
|
else if Length(parts) = 1 then // Edge case: File is "Year.tab" - this is invalid by new rule "Symbol_Year"
|
|
begin SymbolValue := ''; Exit; end // Or handle as Symbol = parts[0] if "Year" itself is a symbol
|
|
else // Should not happen if Length(parts) < 2 already exited
|
|
SymbolValue := '';
|
|
|
|
|
|
Result := True;
|
|
end;
|
|
|
|
function LoadTickDataSeries(const FileName: string): TArray<TTickData>;
|
|
var
|
|
tickList: TList<TTickData>;
|
|
initialYear, currentTabYear, liveProcessingYear: Integer;
|
|
pathName, symbolName: string; // prefixNameOut will be an empty out param
|
|
tabFileName, liveFileName: string;
|
|
overallLastTabTickTime: Double;
|
|
maxTabYearFound: Integer;
|
|
|
|
procedure ReadFileContentAndGetMaxTime(const AFileName: string;
|
|
out Records: TArray<TTickData>;
|
|
out ActualMaxTickTimeInFile: Double);
|
|
var
|
|
fs: TFileStream;
|
|
fileSize: Int64;
|
|
recordCount: Integer;
|
|
bytesRead: Integer;
|
|
iterTickData: TTickData;
|
|
currentFileMaxOA: Double;
|
|
begin
|
|
SetLength(Records, 0);
|
|
ActualMaxTickTimeInFile := 0.0;
|
|
currentFileMaxOA := 0.0;
|
|
|
|
if not TFile.Exists(AFileName) then // Added check for file existence
|
|
begin
|
|
// Silently exit or log if preferred, but don't try to open non-existent file.
|
|
Exit;
|
|
end;
|
|
|
|
fs := TFileStream.Create(AFileName, fmOpenRead or fmShareDenyWrite);
|
|
try
|
|
fileSize := fs.Size;
|
|
if fileSize = 0 then Exit;
|
|
|
|
if (fileSize mod SizeOf(TTickData)) <> 0 then
|
|
raise EReadError.CreateFmt('File %s: Corrupted. Size %d is not a multiple of record size %d.',
|
|
[AFileName, fileSize, SizeOf(TTickData)]);
|
|
|
|
recordCount := fileSize div SizeOf(TTickData);
|
|
if recordCount > 0 then
|
|
begin
|
|
SetLength(Records, recordCount);
|
|
bytesRead := fs.Read(Records[0], fileSize);
|
|
if bytesRead <> fileSize then
|
|
raise EReadError.CreateFmt('File %s: Read error. Expected to read %d bytes, but actually read %d bytes.',
|
|
[AFileName, fileSize, bytesRead]);
|
|
|
|
for iterTickData in Records do
|
|
begin
|
|
if iterTickData.OADateTime > currentFileMaxOA then
|
|
currentFileMaxOA := iterTickData.OADateTime;
|
|
end;
|
|
ActualMaxTickTimeInFile := currentFileMaxOA;
|
|
end;
|
|
finally
|
|
fs.Free;
|
|
end;
|
|
end;
|
|
|
|
begin // Start of LoadTickDataSeries
|
|
tickList := TList<TTickData>.Create;
|
|
overallLastTabTickTime := 0.0;
|
|
maxTabYearFound := -1;
|
|
|
|
try
|
|
// Initial FileName must be the path to the first .tab file (e.g., Symbol_Year.tab)
|
|
// prefixNameOut will be an out parameter from TryParseFileName but will be empty.
|
|
if not TryParseFileName(FileName, pathName, symbolName, initialYear) then
|
|
begin
|
|
raise EArgumentException.CreateFmt('Invalid initial file name format: %s. Expected Symbol_Year.tab (or Symbol_Part2_Year.tab)', [FileName]);
|
|
end;
|
|
|
|
// --- PHASE 1: Load all .tab files and determine overallLastTabTickTime ---
|
|
currentTabYear := initialYear;
|
|
// Construct the first .tab file name using the parsed components (symbol, year).
|
|
// Prefix is no longer used in the format string.
|
|
tabFileName := TPath.Combine(pathName, Format('%s_%d.tab', [symbolName, currentTabYear]));
|
|
|
|
var tabFileRecords: TArray<TTickData>;
|
|
var maxTimeInThisTabFile: Double;
|
|
|
|
while TFile.Exists(tabFileName) do
|
|
begin
|
|
maxTabYearFound := currentTabYear;
|
|
ReadFileContentAndGetMaxTime(tabFileName, tabFileRecords, maxTimeInThisTabFile);
|
|
|
|
if Length(tabFileRecords) > 0 then
|
|
begin
|
|
tickList.AddRange(tabFileRecords);
|
|
end;
|
|
|
|
if maxTimeInThisTabFile > overallLastTabTickTime then
|
|
overallLastTabTickTime := maxTimeInThisTabFile;
|
|
|
|
Inc(currentTabYear);
|
|
// Construct next tab filename without prefix
|
|
tabFileName := TPath.Combine(pathName, Format('%s_%d.tab', [symbolName, currentTabYear]));
|
|
end;
|
|
|
|
// --- PHASE 2: Load .tab-live files ---
|
|
if maxTabYearFound >= initialYear then
|
|
begin
|
|
for liveProcessingYear := initialYear to maxTabYearFound do
|
|
begin
|
|
// Construct .tab-live filename without prefix
|
|
liveFileName := TPath.Combine(pathName, Format('%s_%d.tab-live', [symbolName, liveProcessingYear]));
|
|
|
|
if TFile.Exists(liveFileName) then
|
|
begin
|
|
var liveFileRecords: TArray<TTickData>;
|
|
var maxTimeInThisLiveFile: Double;
|
|
var ticksToActuallyAddFromLive: TList<TTickData>;
|
|
var liveDataAddedThisFile: Boolean;
|
|
var iterLiveTickData: TTickData;
|
|
|
|
ReadFileContentAndGetMaxTime(liveFileName, liveFileRecords, maxTimeInThisLiveFile);
|
|
liveDataAddedThisFile := False;
|
|
|
|
if Length(liveFileRecords) > 0 then
|
|
begin
|
|
ticksToActuallyAddFromLive := TList<TTickData>.Create;
|
|
try
|
|
for iterLiveTickData in liveFileRecords do
|
|
begin
|
|
if iterLiveTickData.OADateTime > overallLastTabTickTime then
|
|
begin
|
|
ticksToActuallyAddFromLive.Add(iterLiveTickData);
|
|
liveDataAddedThisFile := True;
|
|
end;
|
|
end;
|
|
|
|
if ticksToActuallyAddFromLive.Count > 0 then
|
|
tickList.AddRange(ticksToActuallyAddFromLive);
|
|
finally
|
|
ticksToActuallyAddFromLive.Free;
|
|
end;
|
|
end;
|
|
|
|
if not liveDataAddedThisFile then
|
|
begin
|
|
try
|
|
TFile.Delete(liveFileName);
|
|
except
|
|
on E: Exception do {/* Optional: Log error, e.g., using a global logger or passed logger */}
|
|
end;
|
|
end;
|
|
end;
|
|
end;
|
|
end;
|
|
|
|
Result := tickList.ToArray;
|
|
finally
|
|
tickList.Free;
|
|
end;
|
|
end;
|
|
|
|
end.
|