unit Myc.TickLoader;

(*------------------------------------------------------------------------------
  Unit:         Myc.TickLoader
  Author:       Delphi Algo Trading (Assistant)
  Date:         May 22, 2025
  Description:
    This unit provides functions to load tick data (TTickData) from
    binary files. The data originates from a C# application
    and has the structure OADateTime (Double), Ask (Single), and Bid (Single).

    The files now follow the naming convention: Symbol_Year.tab.
    (Prefix has been removed from the convention).
    Loading occurs in two phases:
    1. All .tab files are processed chronologically. Unique ticks are loaded,
       and the timestamp of the latest tick across all .tab files is determined
       (overallLastTabTickTime). Assumes timestamps are unique across all .tab files.
    2. All .tab-live files (for years corresponding to found .tab files) are
       processed. Ticks from a .tab-live file are only integrated if their
       timestamp is strictly later than overallLastTabTickTime. Assumes such
       ticks are globally unique.

    If a .tab-live file results in no new data being added under these rules,
    it is deleted after processing.

    The loading function automatically detects subsequent yearly .tab files.

  Main Function:
    LoadTickDataSeries - Loads a series of tick data files.
------------------------------------------------------------------------------*)

interface

uses
    System.SysUtils,
    System.Classes,
    System.Generics.Collections,
    System.IOUtils;

type
    TTickData = packed record
        OADateTime: Double;
        Ask: Single;
        Bid: Single;
    end;
    PTTickData = ^TTickData;

// TryParseFileName now expects filenames like "Symbol_Year" (or "Symbol_Part2_Year")
// PrefixValue will be returned as an empty string.
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue: Integer): Boolean;
function LoadTickDataSeries(const FileName: string): TArray<TTickData>;

implementation

function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue: Integer): Boolean;
var
    fileNameOnly: string;
    parts: TArray<string>;
begin
    Result := False;
    PathValue := TPath.GetDirectoryName(FileName);
    fileNameOnly := TPath.GetFileNameWithoutExtension(FileName); // Handles any extension for parsing basename

    parts := fileNameOnly.Split(['_']);

    // Expected format is Symbol_Year or SymbolPart1_SymbolPart2_Year, so at least 2 parts.
    if Length(parts) < 2 then
    begin
        SymbolValue := ''; YearValue := 0; // Clear out params on minimum parts failure
        Exit;
    end;

    // Year is the last part.
    if not TryStrToInt(parts[High(parts)], YearValue) then
    begin
        SymbolValue := ''; // Clear SymbolValue as well on year parse failure
        Exit;
    end;

    // Symbol consists of all parts before the year part.
    // Copy all parts except the last one (which is the Year).
    // The second parameter of Copy (for TArray<string>) is the starting index,
    // the third parameter is the count of elements to copy.
    // We want to copy elements from index 0 up to (but not including) the last element.
    // So, the count is High(parts) (which is Length(parts) - 1).
    if High(parts) > 0 then // Ensure there's at least one part for the symbol
        SymbolValue := string.Join('_', Copy(parts, 0, High(parts)))
    else if Length(parts) = 1 then // Edge case: File is "Year.tab" - this is invalid by new rule "Symbol_Year"
         begin SymbolValue := ''; Exit; end // Or handle as Symbol = parts[0] if "Year" itself is a symbol
    else // Should not happen if Length(parts) < 2 already exited
        SymbolValue := '';


    Result := True;
end;

function LoadTickDataSeries(const FileName: string): TArray<TTickData>;
var
    tickList: TList<TTickData>;
    initialYear, currentTabYear, liveProcessingYear: Integer;
    pathName, symbolName: string; // prefixNameOut will be an empty out param
    tabFileName, liveFileName: string;
    overallLastTabTickTime: Double;
    maxTabYearFound: Integer;

    procedure ReadFileContentAndGetMaxTime(const AFileName: string;
                                      out Records: TArray<TTickData>;
                                      out ActualMaxTickTimeInFile: Double);
    var
        fs: TFileStream;
        fileSize: Int64;
        recordCount: Integer;
        bytesRead: Integer;
        iterTickData: TTickData;
        currentFileMaxOA: Double;
    begin
        SetLength(Records, 0);
        ActualMaxTickTimeInFile := 0.0;
        currentFileMaxOA := 0.0;

        if not TFile.Exists(AFileName) then // Added check for file existence
        begin
            // Silently exit or log if preferred, but don't try to open non-existent file.
            Exit;
        end;

        fs := TFileStream.Create(AFileName, fmOpenRead or fmShareDenyWrite);
        try
            fileSize := fs.Size;
            if fileSize = 0 then Exit;

            if (fileSize mod SizeOf(TTickData)) <> 0 then
                raise EReadError.CreateFmt('File %s: Corrupted. Size %d is not a multiple of record size %d.',
                    [AFileName, fileSize, SizeOf(TTickData)]);

            recordCount := fileSize div SizeOf(TTickData);
            if recordCount > 0 then
            begin
                SetLength(Records, recordCount);
                bytesRead := fs.Read(Records[0], fileSize);
                if bytesRead <> fileSize then
                    raise EReadError.CreateFmt('File %s: Read error. Expected to read %d bytes, but actually read %d bytes.',
                        [AFileName, fileSize, bytesRead]);

                for iterTickData in Records do
                begin
                    if iterTickData.OADateTime > currentFileMaxOA then
                        currentFileMaxOA := iterTickData.OADateTime;
                end;
                ActualMaxTickTimeInFile := currentFileMaxOA;
            end;
        finally
            fs.Free;
        end;
    end;

begin // Start of LoadTickDataSeries
    tickList := TList<TTickData>.Create;
    overallLastTabTickTime := 0.0;
    maxTabYearFound := -1;

    try
        // Initial FileName must be the path to the first .tab file (e.g., Symbol_Year.tab)
        // prefixNameOut will be an out parameter from TryParseFileName but will be empty.
        if not TryParseFileName(FileName, pathName, symbolName, initialYear) then
        begin
            raise EArgumentException.CreateFmt('Invalid initial file name format: %s. Expected Symbol_Year.tab (or Symbol_Part2_Year.tab)', [FileName]);
        end;

        // --- PHASE 1: Load all .tab files and determine overallLastTabTickTime ---
        currentTabYear := initialYear;
        // Construct the first .tab file name using the parsed components (symbol, year).
        // Prefix is no longer used in the format string.
        tabFileName := TPath.Combine(pathName, Format('%s_%d.tab', [symbolName, currentTabYear]));

        var tabFileRecords: TArray<TTickData>;
        var maxTimeInThisTabFile: Double;

        while TFile.Exists(tabFileName) do
        begin
            maxTabYearFound := currentTabYear;
            ReadFileContentAndGetMaxTime(tabFileName, tabFileRecords, maxTimeInThisTabFile);

            if Length(tabFileRecords) > 0 then
            begin
                tickList.AddRange(tabFileRecords);
            end;

            if maxTimeInThisTabFile > overallLastTabTickTime then
                overallLastTabTickTime := maxTimeInThisTabFile;

            Inc(currentTabYear);
            // Construct next tab filename without prefix
            tabFileName := TPath.Combine(pathName, Format('%s_%d.tab', [symbolName, currentTabYear]));
        end;

        // --- PHASE 2: Load .tab-live files ---
        if maxTabYearFound >= initialYear then
        begin
            for liveProcessingYear := initialYear to maxTabYearFound do
            begin
                // Construct .tab-live filename without prefix
                liveFileName := TPath.Combine(pathName, Format('%s_%d.tab-live', [symbolName, liveProcessingYear]));

                if TFile.Exists(liveFileName) then
                begin
                    var liveFileRecords: TArray<TTickData>;
                    var maxTimeInThisLiveFile: Double;
                    var ticksToActuallyAddFromLive: TList<TTickData>;
                    var liveDataAddedThisFile: Boolean;
                    var iterLiveTickData: TTickData;

                    ReadFileContentAndGetMaxTime(liveFileName, liveFileRecords, maxTimeInThisLiveFile);
                    liveDataAddedThisFile := False;

                    if Length(liveFileRecords) > 0 then
                    begin
                        ticksToActuallyAddFromLive := TList<TTickData>.Create;
                        try
                            for iterLiveTickData in liveFileRecords do
                            begin
                                if iterLiveTickData.OADateTime > overallLastTabTickTime then
                                begin
                                    ticksToActuallyAddFromLive.Add(iterLiveTickData);
                                    liveDataAddedThisFile := True;
                                end;
                            end;

                            if ticksToActuallyAddFromLive.Count > 0 then
                                tickList.AddRange(ticksToActuallyAddFromLive);
                        finally
                            ticksToActuallyAddFromLive.Free;
                        end;
                    end;

                    if not liveDataAddedThisFile then
                    begin
                        try
                            TFile.Delete(liveFileName);
                        except
                            on E: Exception do {/* Optional: Log error, e.g., using a global logger or passed logger */}
                        end;
                    end;
                end;
            end;
        end;

        Result := tickList.ToArray;
    finally
        tickList.Free;
    end;
end;

end.