1st commit
This commit is contained in:
Michael Schimmel
@@ -0,0 +1,254 @@
|
||||
unit Myc.TickLoader;
|
||||
|
||||
(*------------------------------------------------------------------------------
|
||||
Unit: Myc.TickLoader
|
||||
Author: Delphi Algo Trading (Assistant)
|
||||
Date: May 22, 2025
|
||||
Description:
|
||||
This unit provides functions to load tick data (TTickData) from
|
||||
binary files. The data originates from a C# application
|
||||
and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
|
||||
|
||||
The files now follow the naming convention: Symbol_Year.tab.
|
||||
(Prefix has been removed from the convention).
|
||||
Loading occurs in two phases:
|
||||
1. All .tab files are processed chronologically. Unique ticks are loaded,
|
||||
and the timestamp of the latest tick across all .tab files is determined
|
||||
(overallLastTabTickTime). Assumes timestamps are unique across all .tab files.
|
||||
2. All .tab-live files (for years corresponding to found .tab files) are
|
||||
processed. Ticks from a .tab-live file are only integrated if their
|
||||
timestamp is strictly later than overallLastTabTickTime. Assumes such
|
||||
ticks are globally unique.
|
||||
|
||||
If a .tab-live file results in no new data being added under these rules,
|
||||
it is deleted after processing.
|
||||
|
||||
The loading function automatically detects subsequent yearly .tab files.
|
||||
|
||||
Main Function:
|
||||
LoadTickDataSeries - Loads a series of tick data files.
|
||||
------------------------------------------------------------------------------*)
|
||||
|
||||
interface
|
||||
|
||||
uses
|
||||
System.SysUtils,
|
||||
System.Classes,
|
||||
System.Generics.Collections,
|
||||
System.IOUtils;
|
||||
|
||||
type
|
||||
TTickData = packed record
|
||||
OADateTime: Double;
|
||||
Ask: Single;
|
||||
Bid: Single;
|
||||
end;
|
||||
PTTickData = ^TTickData;
|
||||
|
||||
// TryParseFileName now expects filenames like "Symbol_Year" (or "Symbol_Part2_Year")
|
||||
// PrefixValue will be returned as an empty string.
|
||||
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue: Integer): Boolean;
|
||||
function LoadTickDataSeries(const FileName: string): TArray<TTickData>;
|
||||
|
||||
implementation
|
||||
|
||||
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue: Integer): Boolean;
|
||||
var
|
||||
fileNameOnly: string;
|
||||
parts: TArray<string>;
|
||||
begin
|
||||
Result := False;
|
||||
PathValue := TPath.GetDirectoryName(FileName);
|
||||
fileNameOnly := TPath.GetFileNameWithoutExtension(FileName); // Handles any extension for parsing basename
|
||||
|
||||
parts := fileNameOnly.Split(['_']);
|
||||
|
||||
// Expected format is Symbol_Year or SymbolPart1_SymbolPart2_Year, so at least 2 parts.
|
||||
if Length(parts) < 2 then
|
||||
begin
|
||||
SymbolValue := ''; YearValue := 0; // Clear out params on minimum parts failure
|
||||
Exit;
|
||||
end;
|
||||
|
||||
// Year is the last part.
|
||||
if not TryStrToInt(parts[High(parts)], YearValue) then
|
||||
begin
|
||||
SymbolValue := ''; // Clear SymbolValue as well on year parse failure
|
||||
Exit;
|
||||
end;
|
||||
|
||||
// Symbol consists of all parts before the year part.
|
||||
// Copy all parts except the last one (which is the Year).
|
||||
// The second parameter of Copy (for TArray<string>) is the starting index,
|
||||
// the third parameter is the count of elements to copy.
|
||||
// We want to copy elements from index 0 up to (but not including) the last element.
|
||||
// So, the count is High(parts) (which is Length(parts) - 1).
|
||||
if High(parts) > 0 then // Ensure there's at least one part for the symbol
|
||||
SymbolValue := string.Join('_', Copy(parts, 0, High(parts)))
|
||||
else if Length(parts) = 1 then // Edge case: File is "Year.tab" - this is invalid by new rule "Symbol_Year"
|
||||
begin SymbolValue := ''; Exit; end // Or handle as Symbol = parts[0] if "Year" itself is a symbol
|
||||
else // Should not happen if Length(parts) < 2 already exited
|
||||
SymbolValue := '';
|
||||
|
||||
|
||||
Result := True;
|
||||
end;
|
||||
|
||||
function LoadTickDataSeries(const FileName: string): TArray<TTickData>;
|
||||
var
|
||||
tickList: TList<TTickData>;
|
||||
initialYear, currentTabYear, liveProcessingYear: Integer;
|
||||
pathName, symbolName: string; // prefixNameOut will be an empty out param
|
||||
tabFileName, liveFileName: string;
|
||||
overallLastTabTickTime: Double;
|
||||
maxTabYearFound: Integer;
|
||||
|
||||
procedure ReadFileContentAndGetMaxTime(const AFileName: string;
|
||||
out Records: TArray<TTickData>;
|
||||
out ActualMaxTickTimeInFile: Double);
|
||||
var
|
||||
fs: TFileStream;
|
||||
fileSize: Int64;
|
||||
recordCount: Integer;
|
||||
bytesRead: Integer;
|
||||
iterTickData: TTickData;
|
||||
currentFileMaxOA: Double;
|
||||
begin
|
||||
SetLength(Records, 0);
|
||||
ActualMaxTickTimeInFile := 0.0;
|
||||
currentFileMaxOA := 0.0;
|
||||
|
||||
if not TFile.Exists(AFileName) then // Added check for file existence
|
||||
begin
|
||||
// Silently exit or log if preferred, but don't try to open non-existent file.
|
||||
Exit;
|
||||
end;
|
||||
|
||||
fs := TFileStream.Create(AFileName, fmOpenRead or fmShareDenyWrite);
|
||||
try
|
||||
fileSize := fs.Size;
|
||||
if fileSize = 0 then Exit;
|
||||
|
||||
if (fileSize mod SizeOf(TTickData)) <> 0 then
|
||||
raise EReadError.CreateFmt('File %s: Corrupted. Size %d is not a multiple of record size %d.',
|
||||
[AFileName, fileSize, SizeOf(TTickData)]);
|
||||
|
||||
recordCount := fileSize div SizeOf(TTickData);
|
||||
if recordCount > 0 then
|
||||
begin
|
||||
SetLength(Records, recordCount);
|
||||
bytesRead := fs.Read(Records[0], fileSize);
|
||||
if bytesRead <> fileSize then
|
||||
raise EReadError.CreateFmt('File %s: Read error. Expected to read %d bytes, but actually read %d bytes.',
|
||||
[AFileName, fileSize, bytesRead]);
|
||||
|
||||
for iterTickData in Records do
|
||||
begin
|
||||
if iterTickData.OADateTime > currentFileMaxOA then
|
||||
currentFileMaxOA := iterTickData.OADateTime;
|
||||
end;
|
||||
ActualMaxTickTimeInFile := currentFileMaxOA;
|
||||
end;
|
||||
finally
|
||||
fs.Free;
|
||||
end;
|
||||
end;
|
||||
|
||||
begin // Start of LoadTickDataSeries
|
||||
tickList := TList<TTickData>.Create;
|
||||
overallLastTabTickTime := 0.0;
|
||||
maxTabYearFound := -1;
|
||||
|
||||
try
|
||||
// Initial FileName must be the path to the first .tab file (e.g., Symbol_Year.tab)
|
||||
// prefixNameOut will be an out parameter from TryParseFileName but will be empty.
|
||||
if not TryParseFileName(FileName, pathName, symbolName, initialYear) then
|
||||
begin
|
||||
raise EArgumentException.CreateFmt('Invalid initial file name format: %s. Expected Symbol_Year.tab (or Symbol_Part2_Year.tab)', [FileName]);
|
||||
end;
|
||||
|
||||
// --- PHASE 1: Load all .tab files and determine overallLastTabTickTime ---
|
||||
currentTabYear := initialYear;
|
||||
// Construct the first .tab file name using the parsed components (symbol, year).
|
||||
// Prefix is no longer used in the format string.
|
||||
tabFileName := TPath.Combine(pathName, Format('%s_%d.tab', [symbolName, currentTabYear]));
|
||||
|
||||
var tabFileRecords: TArray<TTickData>;
|
||||
var maxTimeInThisTabFile: Double;
|
||||
|
||||
while TFile.Exists(tabFileName) do
|
||||
begin
|
||||
maxTabYearFound := currentTabYear;
|
||||
ReadFileContentAndGetMaxTime(tabFileName, tabFileRecords, maxTimeInThisTabFile);
|
||||
|
||||
if Length(tabFileRecords) > 0 then
|
||||
begin
|
||||
tickList.AddRange(tabFileRecords);
|
||||
end;
|
||||
|
||||
if maxTimeInThisTabFile > overallLastTabTickTime then
|
||||
overallLastTabTickTime := maxTimeInThisTabFile;
|
||||
|
||||
Inc(currentTabYear);
|
||||
// Construct next tab filename without prefix
|
||||
tabFileName := TPath.Combine(pathName, Format('%s_%d.tab', [symbolName, currentTabYear]));
|
||||
end;
|
||||
|
||||
// --- PHASE 2: Load .tab-live files ---
|
||||
if maxTabYearFound >= initialYear then
|
||||
begin
|
||||
for liveProcessingYear := initialYear to maxTabYearFound do
|
||||
begin
|
||||
// Construct .tab-live filename without prefix
|
||||
liveFileName := TPath.Combine(pathName, Format('%s_%d.tab-live', [symbolName, liveProcessingYear]));
|
||||
|
||||
if TFile.Exists(liveFileName) then
|
||||
begin
|
||||
var liveFileRecords: TArray<TTickData>;
|
||||
var maxTimeInThisLiveFile: Double;
|
||||
var ticksToActuallyAddFromLive: TList<TTickData>;
|
||||
var liveDataAddedThisFile: Boolean;
|
||||
var iterLiveTickData: TTickData;
|
||||
|
||||
ReadFileContentAndGetMaxTime(liveFileName, liveFileRecords, maxTimeInThisLiveFile);
|
||||
liveDataAddedThisFile := False;
|
||||
|
||||
if Length(liveFileRecords) > 0 then
|
||||
begin
|
||||
ticksToActuallyAddFromLive := TList<TTickData>.Create;
|
||||
try
|
||||
for iterLiveTickData in liveFileRecords do
|
||||
begin
|
||||
if iterLiveTickData.OADateTime > overallLastTabTickTime then
|
||||
begin
|
||||
ticksToActuallyAddFromLive.Add(iterLiveTickData);
|
||||
liveDataAddedThisFile := True;
|
||||
end;
|
||||
end;
|
||||
|
||||
if ticksToActuallyAddFromLive.Count > 0 then
|
||||
tickList.AddRange(ticksToActuallyAddFromLive);
|
||||
finally
|
||||
ticksToActuallyAddFromLive.Free;
|
||||
end;
|
||||
end;
|
||||
|
||||
if not liveDataAddedThisFile then
|
||||
begin
|
||||
try
|
||||
TFile.Delete(liveFileName);
|
||||
except
|
||||
on E: Exception do {/* Optional: Log error, e.g., using a global logger or passed logger */}
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
Result := tickList.ToArray;
|
||||
finally
|
||||
tickList.Free;
|
||||
end;
|
||||
end;
|
||||
|
||||
end.
|
||||
Reference in New Issue
Block a user