cTrader-DataExport/common/Myc.TickLoader.pas

unit Myc.TickLoader;

(* ------------------------------------------------------------------------------
  Unit:         Myc.TickLoader
  Author:       Delphi Algo Trading (Assistant)
  Date:         May 27, 2025
  Description:
  This unit provides functions to load tick data (TTickData) from
  binary files. The data originates from a C# application
  and has the structure OADateTime (Double), Ask (Single), and Bid (Single).

  The files follow the naming convention:
  For .tab files:      Symbol_Year_MM.tab OR Symbol_Year_MM.tab_zip
  For .tab-live files: Symbol_Year_MM.tab-live (NEVER compressed)
  (MM represents the two-digit month, e.g., 01 for January, 12 for December)

  If both compressed (.tab_zip) and uncompressed (.tab) versions of a .tab
  file exist, the compressed version is preferred. Files ending with _zip
  are expected to be ZIP archives containing the actual data file
  (e.g., Symbol_Year_MM.tab).

  Loading occurs in two phases:
  1. All .tab files (regular or _zip) are processed chronologically by
  month and year. Unique ticks are loaded, and the timestamp of the
  latest tick across all .tab files is determined (overallLastTabTickTime).
  Assumes timestamps are unique across all .tab files.
  2. All .tab-live files (which are never compressed), for months/years
  corresponding to found .tab files, are processed. Ticks from a
  .tab-live file are only integrated if their timestamp is strictly later
  than overallLastTabTickTime. Assumes such ticks are globally unique.

  If a .tab-live file results in no new data being added under these rules,
  it is deleted after processing.

  The loading function automatically detects subsequent monthly/yearly files.

  Main Function:
  LoadTickDataSeries - Loads a series of tick data files.
  ------------------------------------------------------------------------------ *)

interface

uses
    System.SysUtils,
    System.Classes,
    System.Generics.Collections,
    System.IOUtils,
    Myc.Futures, // Added for TFuture in ReadFileTickData
    Myc.Signals; // Added for IMycState (dependency of TFuture)

type
    TTickData = packed record
        OADateTime: Double;
        Ask: Single;
        Bid: Single;
    end;

    PTTickData = ^TTickData;

    // TryParseFileName parses filenames like "Symbol_Year_MM.Extension" or "Symbol_Year_MM.Extension_zip"
    // to extract Path, Symbol, Year, and Month.
function TryParseFileName( const FileName: string;
    out PathValue, SymbolValue: string;
    out YearValue, MonthValue: Integer ): Boolean;

function LoadTickDataSeries(const FileName: string): TFuture<TArray<TTickData>>;

// Asynchronously reads tick data from a single specified file.
// Returns a TFuture that will provide an array of TTickData.
function ReadFileTickData( var LoadGate: TLatch; const FileName: string ): TFuture<TArray<TTickData>>;

implementation

uses
    System.Zip,
    // Myc.Futures, Myc.Signals already in interface uses for TFuture type visibility
    Myc.TaskManager; // Added for the global TaskManager used by TFuture.Construct

function TryParseFileName( const FileName: string;
    out PathValue, SymbolValue: string;
    out YearValue, MonthValue: Integer ): Boolean;
var
    fileNameNoPath: string;
    nameForParsing: string;
    parts: TArray<string>;
begin
    Result := False; // Default to failure
    PathValue := ''; // Initialize all out parameters
    SymbolValue := '';
    YearValue := 0;
    MonthValue := 0;

    PathValue := TPath.GetDirectoryName( FileName );
    fileNameNoPath := TPath.GetFileName( FileName );

    nameForParsing := fileNameNoPath;
    if nameForParsing.EndsWith( '_zip', True ) then
    begin
        nameForParsing := nameForParsing.Substring( 0, nameForParsing.Length - 4 );
    end;

    nameForParsing := TPath.GetFileNameWithoutExtension( nameForParsing ); // e.g., "EURUSD_2023_01"

    parts := nameForParsing.Split( ['_'] );

    if Length( parts ) < 3 then // Need at least Symbol_Year_Month
    begin
        exit;
    end;

    // Month is the last part.
    if not TryStrToInt( parts[High( parts )], MonthValue ) then
    begin
        exit;
    end;
    if ( MonthValue < 1 ) or ( MonthValue > 12 ) then // Validate month range
    begin
        MonthValue := 0; // Set to invalid if out of range
        exit;
    end;

    // Year is the second to last part.
    if not TryStrToInt( parts[High( parts ) - 1], YearValue ) then
    begin
        MonthValue := 0; // Also invalidate month if year parsing fails
        exit;
    end;

    // Symbol consists of all parts before Year and Month. Count = Length(parts) - 2
    SymbolValue := string.Join( '_', Copy( parts, 0, Length( parts ) - 2 ) );

    if SymbolValue = '' then // Ensure symbol is not empty
    begin
        exit;
    end;

    Result := True;
end;

function LoadTickDataSeries(const FileName: string): TFuture<TArray<TTickData>>;
// Local type declaration for storing file information for phase 1 processing
type
    TPhase1FileEntry = record
        Path: string;
        Year: Integer;
        Month: Integer;
    end;
var
    initialYear, initialMonth: Integer;
    currentTabYear, currentTabMonth: Integer;
    pathName, symbolName: string;
    tabFileEntryRecord: TPhase1FileEntry; // Variable to construct record instances
    loadedState: TState;
    loadedFiles: TArray<TFuture<TArray<TTickData>>>;
    liveData: TFuture<TArray<TTickData>>;

begin // Start of LoadTickDataSeries
    if not TryParseFileName( FileName, pathName, symbolName, initialYear, initialMonth ) then
    begin
        raise EArgumentException.CreateFmt(
            'Invalid initial file name format: %s. Expected Symbol_Year_MM.Extension or Symbol_Year_MM.Extension_zip', [FileName] );
    end;

    currentTabYear := initialYear;
    currentTabMonth := initialMonth;
    var tabBaseNameFormat: string;
    var compressedTabPath, uncompressedTabPath: string;
    var actualTabFileToLoad: string;
    var maxTabYearFound := -1;
    var maxTabMonthFound := -1;

    var filesToLoadPhase1 := TList<TPhase1FileEntry>.Create; // Create list with the named record type
    try
        while True do
        begin
            tabBaseNameFormat := Format( '%s_%.4d_%.2d.tab', [symbolName, currentTabYear, currentTabMonth] );
            compressedTabPath := TPath.Combine( pathName, tabBaseNameFormat + '_zip' );
            uncompressedTabPath := TPath.Combine( pathName, tabBaseNameFormat );

            actualTabFileToLoad := '';
            if TFile.Exists( compressedTabPath ) then
            begin
                actualTabFileToLoad := compressedTabPath;
            end
            else if TFile.Exists( uncompressedTabPath ) then
            begin
                actualTabFileToLoad := uncompressedTabPath;
            end;

            if actualTabFileToLoad = '' then
                break; // No more sequential .tab files

            // Construct the record and add it to the list
            tabFileEntryRecord.Path := actualTabFileToLoad;
            tabFileEntryRecord.Year := currentTabYear;
            tabFileEntryRecord.Month := currentTabMonth;
            filesToLoadPhase1.Add( tabFileEntryRecord );

            // Advance to the next month
            Inc( currentTabMonth );
            if currentTabMonth > 12 then
            begin
                currentTabMonth := 1;
                Inc( currentTabYear );
            end;
        end;

        // Now load and process identified .tab files
        var loadedFileList := TList<TFuture<TArray<TTickData>>>.Create;
        var loadStates := TList<TState>.Create;
        try
            var LoadGate: TLatch;

            for var tabFileEntry: TPhase1FileEntry in filesToLoadPhase1 do // Iterate with the correct type
            begin
                var data := ReadFileTickData( LoadGate, tabFileEntry.Path );

                loadedFileList.Add( TFuture<TArray<TTickData>>.Create( data ) );
                loadStates.Add( data.Done );

                if tabFileEntry.Year > maxTabYearFound then
                begin
                    maxTabYearFound := tabFileEntry.Year;
                    maxTabMonthFound := tabFileEntry.Month;
                end
                else if ( tabFileEntry.Year = maxTabYearFound ) and ( tabFileEntry.Month > maxTabMonthFound ) then
                begin
                    maxTabMonthFound := tabFileEntry.Month;
                end;
            end;

            var liveFileBaseName := Format( '%s_%d_%02d.tab-live', [symbolName, maxTabYearFound, maxTabMonthFound] );
            liveData := ReadFileTickData( LoadGate, TPath.Combine( pathName, liveFileBaseName ) );
            loadStates.Add( liveData.Done );
            loadedFileList.Add( liveData );

            loadedState := TState.All( loadStates.ToArray );
            loadedFiles := loadedFileList.ToArray;
        finally
            loadStates.Free;
            loadedFileList.Free;
        end;
    finally
        filesToLoadPhase1.Free;
    end;

    Result := TFuture<TArray<TTickData>>.Construct(
        loadedState,
        function: TArray<TTickData>
        begin
            var tickList := TList<TTickData>.Create;
            try
                var totalTicks := Length(liveData.Result);
                var overallLastTabTickTime := 0.0;

                for var P in loadedFiles do
                    inc( totalTicks, Length(P.Result) );

                tickList.Capacity := totalTicks;

                for var P in loadedFiles do
                begin
                    if Length(P.Result) > 0 then
                    begin
                        inc( totalTicks, Length(P.Result) );

                        for var iterTickData in P.Result do
                            if iterTickData.OADateTime > overallLastTabTickTime then
                            begin
                                tickList.Add( iterTickData );
                                overallLastTabTickTime := iterTickData.OADateTime;
                            end;
                    end;
                end;

                Result := tickList.ToArray;
            finally
                tickList.Free;
            end;
        end );
end;

function LoadSingleFileTickDataFromStream(const InputStream: TStream): TArray<TTickData>;
var
    fileSize: Int64;
    recordCount: Integer;
    bytesRead: Integer;
begin
    // Initialize Result record
    SetLength(Result, 0);

    InputStream.Position := 0;

    fileSize := InputStream.Size;
    if fileSize = 0 then
        exit;

    if (fileSize mod SizeOf(TTickData)) <> 0 then
        raise EReadError.CreateFmt('File corrupted. Size %d is not a multiple of record size %d.',
            [fileSize, SizeOf(TTickData)]);

    recordCount := fileSize div SizeOf(TTickData);
    if recordCount > 0 then
    begin
        SetLength(Result, recordCount);
        bytesRead := InputStream.Read(Result[0], fileSize);
        if bytesRead <> fileSize then
            raise EReadError.CreateFmt('Read error. Expected to read %d bytes, but actually read %d bytes.',
                [fileSize, bytesRead]);
    end;
end;

function LoadUncompressedSingleFileTickData(const FullFileName: string): TArray<TTickData>;
var
    inputStream: TStream;
begin
    // Initialize Result record
    SetLength(Result, 0);

    if not TFile.Exists(FullFileName) then
    begin
        exit;
    end;

    inputStream := TFileStream.Create(FullFileName, fmOpenRead or fmShareDenyWrite);
    try
        try
            Result := LoadSingleFileTickDataFromStream( inputStream );

        except
            on E: EReadError do
            begin
                raise EReadError.CreateFmt('File %s: %s', [FullFileName, E.Message]);
            end;
        end;
    finally
        inputStream.Free;
    end;
end;

function UncompressTickDataFromStream(const InputStream: TStream): TArray<TTickData>;
var
    decompressionStream: TStream; // Holds uncompressed data of a single zip entry
    localHeader: TZipHeader;
    entryIndex: Integer;
    i: Integer;
    zipFileInstance: TZipFile;
begin
    // Initialize Result record
    SetLength(Result, 0);

    decompressionStream := nil;
    zipFileInstance := nil;

    try
        InputStream.Position := 0; // Reset stream position to the beginning for TZipFile

        // Open TZipFile from this memory stream
        zipFileInstance := TZipFile.Create;
        zipFileInstance.Open(InputStream, TZipMode.zmRead); // Open the zip from the memory stream

        if zipFileInstance.FileCount = 0 then
        begin
            // If TZipFile opens an empty or invalid stream successfully but finds no files
            exit; // FileSuccessfullyLoaded remains False
        end;

        entryIndex := -1;
        for i := 0 to zipFileInstance.FileCount - 1 do
        begin
            if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
            begin
                entryIndex := i;
                break;
            end;
        end;

        if entryIndex = -1 then // If specific name not found, default to the first entry
        begin
            if zipFileInstance.FileCount > 0 then
                entryIndex := 0
            else
                exit; // Should not happen if FileCount > 0 check passed
        end;

        // Decompress the specific entry from the in-memory zip file.
        // TZipFile.Read creates and populates decompressionStream with uncompressed data.
        zipFileInstance.Read(entryIndex, decompressionStream, localHeader);

        if not Assigned(decompressionStream) then
            exit; // Exit if no valid stream could be prepared

        Result := LoadSingleFileTickDataFromStream( decompressionStream );
    finally
        // Ensure all potentially created objects are freed
        if Assigned(decompressionStream) then // Holds uncompressed data from a zip entry
            decompressionStream.Free;
        if Assigned(zipFileInstance) then
            zipFileInstance.Free;
    end;
end;

function ReadFileTickData( var LoadGate: TLatch; const FileName: string ): TFuture<TArray<TTickData>>;
var
    parsedPath, parsedSymbol: string;
    parsedYear, parsedMonth: Integer;
begin
    Result := TFuture<TArray<TTickData>>.Null;

    // Attempt to parse year and month from the filename
    if not TryParseFileName( FileName, parsedPath, parsedSymbol, parsedYear, parsedMonth ) then
    begin
        // If filename parsing fails, return a future that resolves to an empty array.
        exit;
    end;

    if FileName.EndsWith('_zip', True) then
    begin
        var zipFileBytes := TFile.ReadAllBytes(FileName); // Read all bytes of the .zip file
        if Length(zipFileBytes) = 0 then // Handle empty zip file case
            exit;

        var capFileName := FileName;
        var blob := TFuture<TBytes>.Construct(
            TLatch.Enqueue( LoadGate ),
            function: TBytes
            begin
                Result := TFile.ReadAllBytes(capFileName); // Read all bytes of the .zip file
            end );

        Result := blob.Chain<TArray<TTickData>>(
            function( bytes: TBytes ): TArray<TTickData>
            begin
                var zipMemoryStream := TBytesStream.Create( zipFileBytes ); // Create a memory stream to hold the zip file content
                try
                    zipMemoryStream.Position := 0;
                    Result := UncompressTickDataFromStream( zipMemoryStream );
                finally
                    zipMemoryStream.Free;
                end;
            end );
    end
    else
    begin
        var capFileName := FileName;
        Result := TFuture<TArray<TTickData>>.Construct(
            function: TArray<TTickData>
            begin
                Result := LoadUncompressedSingleFileTickData( capFileName );
            end );
    end;
end;

end.