integrated TFuture
This commit is contained in:
Michael Schimmel
+386
-186
@@ -1,33 +1,42 @@
|
||||
unit Myc.TickLoader;
|
||||
|
||||
(*------------------------------------------------------------------------------
|
||||
Unit: Myc.TickLoader
|
||||
Author: Delphi Algo Trading (Assistant)
|
||||
Date: May 22, 2025
|
||||
(* ------------------------------------------------------------------------------
|
||||
Unit: Myc.TickLoader
|
||||
Author: Delphi Algo Trading (Assistant)
|
||||
Date: May 27, 2025
|
||||
Description:
|
||||
This unit provides functions to load tick data (TTickData) from
|
||||
binary files. The data originates from a C# application
|
||||
and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
|
||||
This unit provides functions to load tick data (TTickData) from
|
||||
binary files. The data originates from a C# application
|
||||
and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
|
||||
|
||||
The files now follow the naming convention: Symbol_Year.tab.
|
||||
(Prefix has been removed from the convention).
|
||||
Loading occurs in two phases:
|
||||
1. All .tab files are processed chronologically. Unique ticks are loaded,
|
||||
and the timestamp of the latest tick across all .tab files is determined
|
||||
(overallLastTabTickTime). Assumes timestamps are unique across all .tab files.
|
||||
2. All .tab-live files (for years corresponding to found .tab files) are
|
||||
processed. Ticks from a .tab-live file are only integrated if their
|
||||
timestamp is strictly later than overallLastTabTickTime. Assumes such
|
||||
ticks are globally unique.
|
||||
The files follow the naming convention:
|
||||
For .tab files: Symbol_Year_MM.tab OR Symbol_Year_MM.tab_zip
|
||||
For .tab-live files: Symbol_Year_MM.tab-live (NEVER compressed)
|
||||
(MM represents the two-digit month, e.g., 01 for January, 12 for December)
|
||||
|
||||
If a .tab-live file results in no new data being added under these rules,
|
||||
it is deleted after processing.
|
||||
If both compressed (.tab_zip) and uncompressed (.tab) versions of a .tab
|
||||
file exist, the compressed version is preferred. Files ending with _zip
|
||||
are expected to be ZIP archives containing the actual data file
|
||||
(e.g., Symbol_Year_MM.tab).
|
||||
|
||||
The loading function automatically detects subsequent yearly .tab files.
|
||||
Loading occurs in two phases:
|
||||
1. All .tab files (regular or _zip) are processed chronologically by
|
||||
month and year. Unique ticks are loaded, and the timestamp of the
|
||||
latest tick across all .tab files is determined (overallLastTabTickTime).
|
||||
Assumes timestamps are unique across all .tab files.
|
||||
2. All .tab-live files (which are never compressed), for months/years
|
||||
corresponding to found .tab files, are processed. Ticks from a
|
||||
.tab-live file are only integrated if their timestamp is strictly later
|
||||
than overallLastTabTickTime. Assumes such ticks are globally unique.
|
||||
|
||||
If a .tab-live file results in no new data being added under these rules,
|
||||
it is deleted after processing.
|
||||
|
||||
The loading function automatically detects subsequent monthly/yearly files.
|
||||
|
||||
Main Function:
|
||||
LoadTickDataSeries - Loads a series of tick data files.
|
||||
------------------------------------------------------------------------------*)
|
||||
LoadTickDataSeries - Loads a series of tick data files.
|
||||
------------------------------------------------------------------------------ *)
|
||||
|
||||
interface
|
||||
|
||||
@@ -35,7 +44,9 @@ uses
|
||||
System.SysUtils,
|
||||
System.Classes,
|
||||
System.Generics.Collections,
|
||||
System.IOUtils;
|
||||
System.IOUtils,
|
||||
Myc.Futures, // Added for TFuture in ReadFileTickData
|
||||
Myc.Signals; // Added for IMycState (dependency of TFuture)
|
||||
|
||||
type
|
||||
TTickData = packed record
|
||||
@@ -43,211 +54,400 @@ type
|
||||
Ask: Single;
|
||||
Bid: Single;
|
||||
end;
|
||||
|
||||
PTTickData = ^TTickData;
|
||||
|
||||
// TryParseFileName now expects filenames like "Symbol_Year" (or "Symbol_Part2_Year")
|
||||
// PrefixValue will be returned as an empty string.
|
||||
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue: Integer): Boolean;
|
||||
function LoadTickDataSeries(const FileName: string): TArray<TTickData>;
|
||||
// TryParseFileName parses filenames like "Symbol_Year_MM.Extension" or "Symbol_Year_MM.Extension_zip"
|
||||
// to extract Path, Symbol, Year, and Month.
|
||||
function TryParseFileName( const FileName: string;
|
||||
out PathValue, SymbolValue: string;
|
||||
out YearValue, MonthValue: Integer ): Boolean;
|
||||
|
||||
function LoadTickDataSeries(const FileName: string): TFuture<TArray<TTickData>>;
|
||||
|
||||
// Asynchronously reads tick data from a single specified file.
|
||||
// Returns a TFuture that will provide an array of TTickData.
|
||||
function ReadFileTickData( var LoadGate: TLatch; const FileName: string ): TFuture<TArray<TTickData>>;
|
||||
|
||||
implementation
|
||||
|
||||
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue: Integer): Boolean;
|
||||
uses
|
||||
System.Zip,
|
||||
// Myc.Futures, Myc.Signals already in interface uses for TFuture type visibility
|
||||
Myc.TaskManager; // Added for the global TaskManager used by TFuture.Construct
|
||||
|
||||
function TryParseFileName( const FileName: string;
|
||||
out PathValue, SymbolValue: string;
|
||||
out YearValue, MonthValue: Integer ): Boolean;
|
||||
var
|
||||
fileNameOnly: string;
|
||||
fileNameNoPath: string;
|
||||
nameForParsing: string;
|
||||
parts: TArray<string>;
|
||||
begin
|
||||
Result := False;
|
||||
PathValue := TPath.GetDirectoryName(FileName);
|
||||
fileNameOnly := TPath.GetFileNameWithoutExtension(FileName); // Handles any extension for parsing basename
|
||||
Result := False; // Default to failure
|
||||
PathValue := ''; // Initialize all out parameters
|
||||
SymbolValue := '';
|
||||
YearValue := 0;
|
||||
MonthValue := 0;
|
||||
|
||||
parts := fileNameOnly.Split(['_']);
|
||||
PathValue := TPath.GetDirectoryName( FileName );
|
||||
fileNameNoPath := TPath.GetFileName( FileName );
|
||||
|
||||
// Expected format is Symbol_Year or SymbolPart1_SymbolPart2_Year, so at least 2 parts.
|
||||
if Length(parts) < 2 then
|
||||
nameForParsing := fileNameNoPath;
|
||||
if nameForParsing.EndsWith( '_zip', True ) then
|
||||
begin
|
||||
SymbolValue := ''; YearValue := 0; // Clear out params on minimum parts failure
|
||||
Exit;
|
||||
nameForParsing := nameForParsing.Substring( 0, nameForParsing.Length - 4 );
|
||||
end;
|
||||
|
||||
// Year is the last part.
|
||||
if not TryStrToInt(parts[High(parts)], YearValue) then
|
||||
nameForParsing := TPath.GetFileNameWithoutExtension( nameForParsing ); // e.g., "EURUSD_2023_01"
|
||||
|
||||
parts := nameForParsing.Split( ['_'] );
|
||||
|
||||
if Length( parts ) < 3 then // Need at least Symbol_Year_Month
|
||||
begin
|
||||
SymbolValue := ''; // Clear SymbolValue as well on year parse failure
|
||||
Exit;
|
||||
exit;
|
||||
end;
|
||||
|
||||
// Symbol consists of all parts before the year part.
|
||||
// Copy all parts except the last one (which is the Year).
|
||||
// The second parameter of Copy (for TArray<string>) is the starting index,
|
||||
// the third parameter is the count of elements to copy.
|
||||
// We want to copy elements from index 0 up to (but not including) the last element.
|
||||
// So, the count is High(parts) (which is Length(parts) - 1).
|
||||
if High(parts) > 0 then // Ensure there's at least one part for the symbol
|
||||
SymbolValue := string.Join('_', Copy(parts, 0, High(parts)))
|
||||
else if Length(parts) = 1 then // Edge case: File is "Year.tab" - this is invalid by new rule "Symbol_Year"
|
||||
begin SymbolValue := ''; Exit; end // Or handle as Symbol = parts[0] if "Year" itself is a symbol
|
||||
else // Should not happen if Length(parts) < 2 already exited
|
||||
SymbolValue := '';
|
||||
// Month is the last part.
|
||||
if not TryStrToInt( parts[High( parts )], MonthValue ) then
|
||||
begin
|
||||
exit;
|
||||
end;
|
||||
if ( MonthValue < 1 ) or ( MonthValue > 12 ) then // Validate month range
|
||||
begin
|
||||
MonthValue := 0; // Set to invalid if out of range
|
||||
exit;
|
||||
end;
|
||||
|
||||
// Year is the second to last part.
|
||||
if not TryStrToInt( parts[High( parts ) - 1], YearValue ) then
|
||||
begin
|
||||
MonthValue := 0; // Also invalidate month if year parsing fails
|
||||
exit;
|
||||
end;
|
||||
|
||||
// Symbol consists of all parts before Year and Month. Count = Length(parts) - 2
|
||||
SymbolValue := string.Join( '_', Copy( parts, 0, Length( parts ) - 2 ) );
|
||||
|
||||
if SymbolValue = '' then // Ensure symbol is not empty
|
||||
begin
|
||||
exit;
|
||||
end;
|
||||
|
||||
Result := True;
|
||||
end;
|
||||
|
||||
function LoadTickDataSeries(const FileName: string): TArray<TTickData>;
|
||||
var
|
||||
tickList: TList<TTickData>;
|
||||
initialYear, currentTabYear, liveProcessingYear: Integer;
|
||||
pathName, symbolName: string; // prefixNameOut will be an empty out param
|
||||
tabFileName, liveFileName: string;
|
||||
overallLastTabTickTime: Double;
|
||||
maxTabYearFound: Integer;
|
||||
|
||||
procedure ReadFileContentAndGetMaxTime(const AFileName: string;
|
||||
out Records: TArray<TTickData>;
|
||||
out ActualMaxTickTimeInFile: Double);
|
||||
var
|
||||
fs: TFileStream;
|
||||
fileSize: Int64;
|
||||
recordCount: Integer;
|
||||
bytesRead: Integer;
|
||||
iterTickData: TTickData;
|
||||
currentFileMaxOA: Double;
|
||||
begin
|
||||
SetLength(Records, 0);
|
||||
ActualMaxTickTimeInFile := 0.0;
|
||||
currentFileMaxOA := 0.0;
|
||||
|
||||
if not TFile.Exists(AFileName) then // Added check for file existence
|
||||
begin
|
||||
// Silently exit or log if preferred, but don't try to open non-existent file.
|
||||
Exit;
|
||||
end;
|
||||
|
||||
fs := TFileStream.Create(AFileName, fmOpenRead or fmShareDenyWrite);
|
||||
try
|
||||
fileSize := fs.Size;
|
||||
if fileSize = 0 then Exit;
|
||||
|
||||
if (fileSize mod SizeOf(TTickData)) <> 0 then
|
||||
raise EReadError.CreateFmt('File %s: Corrupted. Size %d is not a multiple of record size %d.',
|
||||
[AFileName, fileSize, SizeOf(TTickData)]);
|
||||
|
||||
recordCount := fileSize div SizeOf(TTickData);
|
||||
if recordCount > 0 then
|
||||
begin
|
||||
SetLength(Records, recordCount);
|
||||
bytesRead := fs.Read(Records[0], fileSize);
|
||||
if bytesRead <> fileSize then
|
||||
raise EReadError.CreateFmt('File %s: Read error. Expected to read %d bytes, but actually read %d bytes.',
|
||||
[AFileName, fileSize, bytesRead]);
|
||||
|
||||
for iterTickData in Records do
|
||||
begin
|
||||
if iterTickData.OADateTime > currentFileMaxOA then
|
||||
currentFileMaxOA := iterTickData.OADateTime;
|
||||
end;
|
||||
ActualMaxTickTimeInFile := currentFileMaxOA;
|
||||
end;
|
||||
finally
|
||||
fs.Free;
|
||||
end;
|
||||
function LoadTickDataSeries(const FileName: string): TFuture<TArray<TTickData>>;
|
||||
// Local type declaration for storing file information for phase 1 processing
|
||||
type
|
||||
TPhase1FileEntry = record
|
||||
Path: string;
|
||||
Year: Integer;
|
||||
Month: Integer;
|
||||
end;
|
||||
var
|
||||
initialYear, initialMonth: Integer;
|
||||
currentTabYear, currentTabMonth: Integer;
|
||||
pathName, symbolName: string;
|
||||
tabFileEntryRecord: TPhase1FileEntry; // Variable to construct record instances
|
||||
loadedState: TState;
|
||||
loadedFiles: TArray<TFuture<TArray<TTickData>>>;
|
||||
liveData: TFuture<TArray<TTickData>>;
|
||||
|
||||
begin // Start of LoadTickDataSeries
|
||||
tickList := TList<TTickData>.Create;
|
||||
overallLastTabTickTime := 0.0;
|
||||
maxTabYearFound := -1;
|
||||
if not TryParseFileName( FileName, pathName, symbolName, initialYear, initialMonth ) then
|
||||
begin
|
||||
raise EArgumentException.CreateFmt(
|
||||
'Invalid initial file name format: %s. Expected Symbol_Year_MM.Extension or Symbol_Year_MM.Extension_zip', [FileName] );
|
||||
end;
|
||||
|
||||
currentTabYear := initialYear;
|
||||
currentTabMonth := initialMonth;
|
||||
var tabBaseNameFormat: string;
|
||||
var compressedTabPath, uncompressedTabPath: string;
|
||||
var actualTabFileToLoad: string;
|
||||
var maxTabYearFound := -1;
|
||||
var maxTabMonthFound := -1;
|
||||
|
||||
var filesToLoadPhase1 := TList<TPhase1FileEntry>.Create; // Create list with the named record type
|
||||
try
|
||||
// Initial FileName must be the path to the first .tab file (e.g., Symbol_Year.tab)
|
||||
// prefixNameOut will be an out parameter from TryParseFileName but will be empty.
|
||||
if not TryParseFileName(FileName, pathName, symbolName, initialYear) then
|
||||
while True do
|
||||
begin
|
||||
raise EArgumentException.CreateFmt('Invalid initial file name format: %s. Expected Symbol_Year.tab (or Symbol_Part2_Year.tab)', [FileName]);
|
||||
end;
|
||||
tabBaseNameFormat := Format( '%s_%.4d_%.2d.tab', [symbolName, currentTabYear, currentTabMonth] );
|
||||
compressedTabPath := TPath.Combine( pathName, tabBaseNameFormat + '_zip' );
|
||||
uncompressedTabPath := TPath.Combine( pathName, tabBaseNameFormat );
|
||||
|
||||
// --- PHASE 1: Load all .tab files and determine overallLastTabTickTime ---
|
||||
currentTabYear := initialYear;
|
||||
// Construct the first .tab file name using the parsed components (symbol, year).
|
||||
// Prefix is no longer used in the format string.
|
||||
tabFileName := TPath.Combine(pathName, Format('%s_%d.tab', [symbolName, currentTabYear]));
|
||||
|
||||
var tabFileRecords: TArray<TTickData>;
|
||||
var maxTimeInThisTabFile: Double;
|
||||
|
||||
while TFile.Exists(tabFileName) do
|
||||
begin
|
||||
maxTabYearFound := currentTabYear;
|
||||
ReadFileContentAndGetMaxTime(tabFileName, tabFileRecords, maxTimeInThisTabFile);
|
||||
|
||||
if Length(tabFileRecords) > 0 then
|
||||
actualTabFileToLoad := '';
|
||||
if TFile.Exists( compressedTabPath ) then
|
||||
begin
|
||||
tickList.AddRange(tabFileRecords);
|
||||
actualTabFileToLoad := compressedTabPath;
|
||||
end
|
||||
else if TFile.Exists( uncompressedTabPath ) then
|
||||
begin
|
||||
actualTabFileToLoad := uncompressedTabPath;
|
||||
end;
|
||||
|
||||
if maxTimeInThisTabFile > overallLastTabTickTime then
|
||||
overallLastTabTickTime := maxTimeInThisTabFile;
|
||||
if actualTabFileToLoad = '' then
|
||||
break; // No more sequential .tab files
|
||||
|
||||
Inc(currentTabYear);
|
||||
// Construct next tab filename without prefix
|
||||
tabFileName := TPath.Combine(pathName, Format('%s_%d.tab', [symbolName, currentTabYear]));
|
||||
// Construct the record and add it to the list
|
||||
tabFileEntryRecord.Path := actualTabFileToLoad;
|
||||
tabFileEntryRecord.Year := currentTabYear;
|
||||
tabFileEntryRecord.Month := currentTabMonth;
|
||||
filesToLoadPhase1.Add( tabFileEntryRecord );
|
||||
|
||||
// Advance to the next month
|
||||
Inc( currentTabMonth );
|
||||
if currentTabMonth > 12 then
|
||||
begin
|
||||
currentTabMonth := 1;
|
||||
Inc( currentTabYear );
|
||||
end;
|
||||
end;
|
||||
|
||||
// --- PHASE 2: Load .tab-live files ---
|
||||
if maxTabYearFound >= initialYear then
|
||||
begin
|
||||
for liveProcessingYear := initialYear to maxTabYearFound do
|
||||
// Now load and process identified .tab files
|
||||
var loadedFileList := TList<TFuture<TArray<TTickData>>>.Create;
|
||||
var loadStates := TList<TState>.Create;
|
||||
try
|
||||
var LoadGate: TLatch;
|
||||
|
||||
for var tabFileEntry: TPhase1FileEntry in filesToLoadPhase1 do // Iterate with the correct type
|
||||
begin
|
||||
// Construct .tab-live filename without prefix
|
||||
liveFileName := TPath.Combine(pathName, Format('%s_%d.tab-live', [symbolName, liveProcessingYear]));
|
||||
var data := ReadFileTickData( LoadGate, tabFileEntry.Path );
|
||||
|
||||
if TFile.Exists(liveFileName) then
|
||||
loadedFileList.Add( TFuture<TArray<TTickData>>.Create( data ) );
|
||||
loadStates.Add( data.Done );
|
||||
|
||||
if tabFileEntry.Year > maxTabYearFound then
|
||||
begin
|
||||
var liveFileRecords: TArray<TTickData>;
|
||||
var maxTimeInThisLiveFile: Double;
|
||||
var ticksToActuallyAddFromLive: TList<TTickData>;
|
||||
var liveDataAddedThisFile: Boolean;
|
||||
var iterLiveTickData: TTickData;
|
||||
|
||||
ReadFileContentAndGetMaxTime(liveFileName, liveFileRecords, maxTimeInThisLiveFile);
|
||||
liveDataAddedThisFile := False;
|
||||
|
||||
if Length(liveFileRecords) > 0 then
|
||||
begin
|
||||
ticksToActuallyAddFromLive := TList<TTickData>.Create;
|
||||
try
|
||||
for iterLiveTickData in liveFileRecords do
|
||||
begin
|
||||
if iterLiveTickData.OADateTime > overallLastTabTickTime then
|
||||
begin
|
||||
ticksToActuallyAddFromLive.Add(iterLiveTickData);
|
||||
liveDataAddedThisFile := True;
|
||||
end;
|
||||
end;
|
||||
|
||||
if ticksToActuallyAddFromLive.Count > 0 then
|
||||
tickList.AddRange(ticksToActuallyAddFromLive);
|
||||
finally
|
||||
ticksToActuallyAddFromLive.Free;
|
||||
end;
|
||||
end;
|
||||
|
||||
if not liveDataAddedThisFile then
|
||||
begin
|
||||
try
|
||||
TFile.Delete(liveFileName);
|
||||
except
|
||||
on E: Exception do {/* Optional: Log error, e.g., using a global logger or passed logger */}
|
||||
end;
|
||||
end;
|
||||
maxTabYearFound := tabFileEntry.Year;
|
||||
maxTabMonthFound := tabFileEntry.Month;
|
||||
end
|
||||
else if ( tabFileEntry.Year = maxTabYearFound ) and ( tabFileEntry.Month > maxTabMonthFound ) then
|
||||
begin
|
||||
maxTabMonthFound := tabFileEntry.Month;
|
||||
end;
|
||||
end;
|
||||
|
||||
var liveFileBaseName := Format( '%s_%d_%02d.tab-live', [symbolName, maxTabYearFound, maxTabMonthFound] );
|
||||
liveData := ReadFileTickData( LoadGate, TPath.Combine( pathName, liveFileBaseName ) );
|
||||
loadStates.Add( liveData.Done );
|
||||
loadedFileList.Add( liveData );
|
||||
|
||||
loadedState := TState.All( loadStates.ToArray );
|
||||
loadedFiles := loadedFileList.ToArray;
|
||||
finally
|
||||
loadStates.Free;
|
||||
loadedFileList.Free;
|
||||
end;
|
||||
finally
|
||||
filesToLoadPhase1.Free;
|
||||
end;
|
||||
|
||||
Result := TFuture<TArray<TTickData>>.Construct(
|
||||
loadedState,
|
||||
function: TArray<TTickData>
|
||||
begin
|
||||
var tickList := TList<TTickData>.Create;
|
||||
try
|
||||
var totalTicks := Length(liveData.Result);
|
||||
var overallLastTabTickTime := 0.0;
|
||||
|
||||
for var P in loadedFiles do
|
||||
inc( totalTicks, Length(P.Result) );
|
||||
|
||||
tickList.Capacity := totalTicks;
|
||||
|
||||
for var P in loadedFiles do
|
||||
begin
|
||||
if Length(P.Result) > 0 then
|
||||
begin
|
||||
inc( totalTicks, Length(P.Result) );
|
||||
|
||||
for var iterTickData in P.Result do
|
||||
if iterTickData.OADateTime > overallLastTabTickTime then
|
||||
begin
|
||||
tickList.Add( iterTickData );
|
||||
overallLastTabTickTime := iterTickData.OADateTime;
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
Result := tickList.ToArray;
|
||||
finally
|
||||
tickList.Free;
|
||||
end;
|
||||
end );
|
||||
end;
|
||||
|
||||
function LoadSingleFileTickDataFromStream(const InputStream: TStream): TArray<TTickData>;
|
||||
var
|
||||
fileSize: Int64;
|
||||
recordCount: Integer;
|
||||
bytesRead: Integer;
|
||||
begin
|
||||
// Initialize Result record
|
||||
SetLength(Result, 0);
|
||||
|
||||
InputStream.Position := 0;
|
||||
|
||||
fileSize := InputStream.Size;
|
||||
if fileSize = 0 then
|
||||
exit;
|
||||
|
||||
if (fileSize mod SizeOf(TTickData)) <> 0 then
|
||||
raise EReadError.CreateFmt('File corrupted. Size %d is not a multiple of record size %d.',
|
||||
[fileSize, SizeOf(TTickData)]);
|
||||
|
||||
recordCount := fileSize div SizeOf(TTickData);
|
||||
if recordCount > 0 then
|
||||
begin
|
||||
SetLength(Result, recordCount);
|
||||
bytesRead := InputStream.Read(Result[0], fileSize);
|
||||
if bytesRead <> fileSize then
|
||||
raise EReadError.CreateFmt('Read error. Expected to read %d bytes, but actually read %d bytes.',
|
||||
[fileSize, bytesRead]);
|
||||
end;
|
||||
end;
|
||||
|
||||
function LoadUncompressedSingleFileTickData(const FullFileName: string): TArray<TTickData>;
|
||||
var
|
||||
inputStream: TStream;
|
||||
begin
|
||||
// Initialize Result record
|
||||
SetLength(Result, 0);
|
||||
|
||||
if not TFile.Exists(FullFileName) then
|
||||
begin
|
||||
exit;
|
||||
end;
|
||||
|
||||
inputStream := TFileStream.Create(FullFileName, fmOpenRead or fmShareDenyWrite);
|
||||
try
|
||||
try
|
||||
Result := LoadSingleFileTickDataFromStream( inputStream );
|
||||
|
||||
except
|
||||
on E: EReadError do
|
||||
begin
|
||||
raise EReadError.CreateFmt('File %s: %s', [FullFileName, E.Message]);
|
||||
end;
|
||||
end;
|
||||
finally
|
||||
inputStream.Free;
|
||||
end;
|
||||
end;
|
||||
|
||||
function UncompressTickDataFromStream(const InputStream: TStream): TArray<TTickData>;
|
||||
var
|
||||
decompressionStream: TStream; // Holds uncompressed data of a single zip entry
|
||||
localHeader: TZipHeader;
|
||||
entryIndex: Integer;
|
||||
i: Integer;
|
||||
zipFileInstance: TZipFile;
|
||||
begin
|
||||
// Initialize Result record
|
||||
SetLength(Result, 0);
|
||||
|
||||
decompressionStream := nil;
|
||||
zipFileInstance := nil;
|
||||
|
||||
try
|
||||
InputStream.Position := 0; // Reset stream position to the beginning for TZipFile
|
||||
|
||||
// Open TZipFile from this memory stream
|
||||
zipFileInstance := TZipFile.Create;
|
||||
zipFileInstance.Open(InputStream, TZipMode.zmRead); // Open the zip from the memory stream
|
||||
|
||||
if zipFileInstance.FileCount = 0 then
|
||||
begin
|
||||
// If TZipFile opens an empty or invalid stream successfully but finds no files
|
||||
exit; // FileSuccessfullyLoaded remains False
|
||||
end;
|
||||
|
||||
Result := tickList.ToArray;
|
||||
entryIndex := -1;
|
||||
for i := 0 to zipFileInstance.FileCount - 1 do
|
||||
begin
|
||||
if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
|
||||
begin
|
||||
entryIndex := i;
|
||||
break;
|
||||
end;
|
||||
end;
|
||||
|
||||
if entryIndex = -1 then // If specific name not found, default to the first entry
|
||||
begin
|
||||
if zipFileInstance.FileCount > 0 then
|
||||
entryIndex := 0
|
||||
else
|
||||
exit; // Should not happen if FileCount > 0 check passed
|
||||
end;
|
||||
|
||||
// Decompress the specific entry from the in-memory zip file.
|
||||
// TZipFile.Read creates and populates decompressionStream with uncompressed data.
|
||||
zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
|
||||
|
||||
if not Assigned(decompressionStream) then
|
||||
exit; // Exit if no valid stream could be prepared
|
||||
|
||||
Result := LoadSingleFileTickDataFromStream( decompressionStream );
|
||||
finally
|
||||
tickList.Free;
|
||||
// Ensure all potentially created objects are freed
|
||||
if Assigned(decompressionStream) then // Holds uncompressed data from a zip entry
|
||||
decompressionStream.Free;
|
||||
if Assigned(zipFileInstance) then
|
||||
zipFileInstance.Free;
|
||||
end;
|
||||
end;
|
||||
|
||||
function ReadFileTickData( var LoadGate: TLatch; const FileName: string ): TFuture<TArray<TTickData>>;
|
||||
var
|
||||
parsedPath, parsedSymbol: string;
|
||||
parsedYear, parsedMonth: Integer;
|
||||
begin
|
||||
Result := TFuture<TArray<TTickData>>.Null;
|
||||
|
||||
// Attempt to parse year and month from the filename
|
||||
if not TryParseFileName( FileName, parsedPath, parsedSymbol, parsedYear, parsedMonth ) then
|
||||
begin
|
||||
// If filename parsing fails, return a future that resolves to an empty array.
|
||||
exit;
|
||||
end;
|
||||
|
||||
if FileName.EndsWith('_zip', True) then
|
||||
begin
|
||||
var zipFileBytes := TFile.ReadAllBytes(FileName); // Read all bytes of the .zip file
|
||||
if Length(zipFileBytes) = 0 then // Handle empty zip file case
|
||||
exit;
|
||||
|
||||
var capFileName := FileName;
|
||||
var blob := TFuture<TBytes>.Construct(
|
||||
TLatch.Enqueue( LoadGate ),
|
||||
function: TBytes
|
||||
begin
|
||||
Result := TFile.ReadAllBytes(capFileName); // Read all bytes of the .zip file
|
||||
end );
|
||||
|
||||
Result := blob.Chain<TArray<TTickData>>(
|
||||
function( bytes: TBytes ): TArray<TTickData>
|
||||
begin
|
||||
var zipMemoryStream := TBytesStream.Create( zipFileBytes ); // Create a memory stream to hold the zip file content
|
||||
try
|
||||
zipMemoryStream.Position := 0;
|
||||
Result := UncompressTickDataFromStream( zipMemoryStream );
|
||||
finally
|
||||
zipMemoryStream.Free;
|
||||
end;
|
||||
end );
|
||||
end
|
||||
else
|
||||
begin
|
||||
var capFileName := FileName;
|
||||
Result := TFuture<TArray<TTickData>>.Construct(
|
||||
function: TArray<TTickData>
|
||||
begin
|
||||
Result := LoadUncompressedSingleFileTickData( capFileName );
|
||||
end );
|
||||
end;
|
||||
end;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user