unit Myc.Trade.DataSeries;

(* ------------------------------------------------------------------------------
  This unit provides functions to load tick data (TTickData) from
  binary files. The data originates from a C# application
  and has the structure OADateTime (Double), Ask (Single), and Bid (Single).

  The files follow the naming convention:
  For .tab files:     Symbol_Year_MM.tab OR Symbol_Year_MM.tab_zip
  For .tab-live files: Symbol_Year_MM.tab-live (NEVER compressed)
  (MM represents the two-digit month, e.g., 01 for January, 12 for December)

  If both compressed (.tab_zip) and uncompressed (.tab) versions of a .tab
  file exist, the compressed version is preferred. Files ending with _zip
  are expected to be ZIP archives containing the actual data file
  (e.g., Symbol_Year_MM.tab).

  Loading occurs in two phases:
  1. All .tab files (regular or _zip) are processed chronologically by
  month and year. Unique ticks are loaded, and the timestamp of the
  latest tick across all .tab files is determined (overallLastTabTickTime).
  Assumes timestamps are unique across all .tab files.
  2. All .tab-live files (which are never compressed), for months/years
  corresponding to found .tab files, are processed. Ticks from a
  .tab-live file are only integrated if their timestamp is strictly later
  than overallLastTabTickTime. Assumes such ticks are globally unique.

  If a .tab-live file results in no new data being added under these rules,
  it is deleted after processing.

  The loading function automatically detects subsequent monthly/yearly files.

  Main Function:
  LoadTickDataSeries - Loads a series of tick data files.
  ------------------------------------------------------------------------------ *)

interface

uses
    System.SysUtils,
    System.Classes,
    System.Generics.Collections,
    System.Generics.Defaults,
    System.IOUtils,
    Myc.Futures,
    Myc.Signals;

type
    TDataSeries<T: record> = class
        type
            TDataPoint = packed record
                OADateTime: Double;
                Data: T;
            end;

    strict private
        type
            TCachedFile = record
                Name: String;
                Age: TDateTime;
                LastUsed: TDateTime;
                Data: TFuture<TArray<TDataPoint>>;
            end;
        class var
            FCachedFiles: TDictionary<String, TCachedFile>;

        class constructor CreateClass;
        class destructor DestroyClass;

    protected
        class function ReadCompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
        class function ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
    public
        class function LoadDataFile(var LoadGate: TLatch; const FileName: string): TFuture<TArray<TDataPoint>>; static;
        class function LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint>>; static;
    end;

    TAskBidItem = packed record
        Ask: Single;
        Bid: Single;
    end;

    TAskBid = class(TDataSeries<TAskBidItem>)
        type
            TTick = TDataSeries<TAskBidItem>.TDataPoint;
    end;

function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue, MonthValue: Integer): Boolean;

// Finds the full filename of the oldest (=first) file for each symbol in the path.
function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;

var
    IsMemoryLow: TFunc<Boolean>;

implementation

uses
    System.Zip,
    System.Math,
    Myc.TaskManager;

// TryParseFileName parses filenames like "Symbol_Year_MM.Extension" or "Symbol_Year_MM.Extension_zip"
// to extract Path, Symbol, Year, and Month.
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue, MonthValue: Integer): Boolean;
var
    fileNameNoPath: string;
    nameForParsing: string;
    parts: TArray<string>;
    baseName: string;
begin
    Result := False; // Default to failure
    PathValue := ''; // Initialize all out parameters
    SymbolValue := '';
    YearValue := 0;
    MonthValue := 0;

    PathValue := TPath.GetDirectoryName(FileName);
    fileNameNoPath := TPath.GetFileName(FileName);

    nameForParsing := fileNameNoPath;
    // Handle cases like "Symbol_Year_MM.Extension_zip" by removing the trailing "_zip" before extension removal
    if nameForParsing.EndsWith('_zip', True) then
    begin
        // Get the part before "_zip"
        baseName := nameForParsing.Substring(0, nameForParsing.Length - 4);
        // Get the extension of the original name (e.g. ".csv_zip")
        // and check if baseName still contains an extension that needs to be removed by GetFileNameWithoutExtension
        if TPath.GetExtension(baseName) <> '' then
        begin
            nameForParsing := baseName;
        end
        else
        begin
            // If baseName (e.g. "EURUSD_2023_01") has no extension, it is the correct parsing base
            // However, TPath.GetFileNameWithoutExtension would also work correctly here,
            // but this handles a potential case where a file might be named "Symbol_Year_MM_zip" (no dot before zip)
        end;
    end;

    // Remove the actual extension (e.g., ".csv", ".dat") from "Symbol_Year_MM.Extension" or the modified "Symbol_Year_MM.Extension" part of "_zip"
    nameForParsing := TPath.GetFileNameWithoutExtension(nameForParsing); // e.g., "EURUSD_2023_01"

    parts := nameForParsing.Split(['_']);

    if Length(parts) < 3 then // Need at least Symbol_Year_Month
        exit;

    // Month is the last part.
    if not TryStrToInt(parts[High(parts)], MonthValue) then
    begin
        exit;
    end;

    if (MonthValue < 1) or (MonthValue > 12) then // Validate month range
    begin
        MonthValue := 0; // Set to invalid if out of range
        exit;
    end;

    // Year is the second to last part.
    if not TryStrToInt(parts[High(parts) - 1], YearValue) then
    begin
        MonthValue := 0; // Also invalidate month if year parsing fails
        exit;
    end;
    // Ensure YearValue is plausible, e.g. not 0 or negative, adjust as needed for your data's year range
    if YearValue <= 0 then
    begin
        YearValue := 0;
        MonthValue := 0;
        exit;
    end;

    // Symbol consists of all parts before Year and Month.
    SymbolValue := string.Join('_', Copy(parts, 0, Length(parts) - 2));

    if SymbolValue = '' then // Ensure symbol is not empty
    begin
        // Reset all out parameters on failure before exit
        PathValue := '';
        YearValue := 0;
        MonthValue := 0;
        exit;
    end;

    Result := True;
end;

// Finds the oldest file for each symbol in the given directory.
// The file extension is ignored for the purpose of identifying the symbol, year, and month.
function FindOldestFilesPerSymbol(const DirectoryPath: string): TArray<String>;
type
    TTrackedFileInfo = record
        FullFileName: string;
        Year: Integer;
        Month: Integer;
    end;
var
    oldestFilesPerSymbol: TDictionary<string, TTrackedFileInfo>;
    fileNames: TArray<string>;
    currentFile: string;
    parsedPath: string;
    parsedSymbol: string;
    parsedYear: Integer;
    parsedMonth: Integer;
    trackedInfo: TTrackedFileInfo;
begin
    oldestFilesPerSymbol := TDictionary<string, TTrackedFileInfo>.Create; // Dictionary to track the oldest file per symbol
    try
        if not TDirectory.Exists(DirectoryPath) then
        begin
            // Or raise an exception, depending on desired error handling
            exit;
        end;

        fileNames := TDirectory.GetFiles(DirectoryPath); // Get all files in the directory

        for currentFile in fileNames do
        begin
            // Attempt to parse the file name
            if TryParseFileName(currentFile, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
            begin
                // Check if this symbol is already tracked or if this file is older
                if oldestFilesPerSymbol.TryGetValue(parsedSymbol, trackedInfo) then
                begin
                    // If current file's year is less, or year is same and month is less, it's older
                    if (parsedYear < trackedInfo.Year) or ((parsedYear = trackedInfo.Year) and (parsedMonth < trackedInfo.Month)) then
                    begin
                        trackedInfo.FullFileName := currentFile; // Store full path to the file
                        trackedInfo.Year := parsedYear;
                        trackedInfo.Month := parsedMonth;
                        oldestFilesPerSymbol.AddOrSetValue(parsedSymbol, trackedInfo); // Update the entry for this symbol
                    end;
                end
                else
                begin
                    // First time we see this symbol, so it's the oldest so far
                    trackedInfo.FullFileName := currentFile; // Store full path to the file
                    trackedInfo.Year := parsedYear;
                    trackedInfo.Month := parsedMonth;
                    oldestFilesPerSymbol.Add(parsedSymbol, trackedInfo); // Add new entry for this symbol
                end;
            end;
        end;

        // Populate the result array with the full names of the oldest files
        SetLength(Result, oldestFilesPerSymbol.Count);
        var n := 0;
        for trackedInfo in oldestFilesPerSymbol.Values do
        begin
            Result[n] := trackedInfo.FullFileName;
            inc(n);
        end;
    finally
        oldestFilesPerSymbol.Free; // Free the dictionary
    end;
end;

class destructor TDataSeries<T>.CreateClass;
begin
    FCachedFiles := TDictionary<String, TCachedFile>.Create;
end;

class destructor TDataSeries<T>.DestroyClass;
begin
    FCachedFiles.Free;
end;

// Asynchronously reads tick data from a single specified file.
// Returns a TFuture that will provide an array of TAskBidSeries.
class function TDataSeries<T>.LoadDataFile(var LoadGate: TLatch; const FileName: string): TFuture<TArray<TDataPoint>>;
var
    parsedPath, parsedSymbol: string;
    parsedYear, parsedMonth: Integer;
begin
    Result := TFuture<TArray<TDataPoint>>.Null;

    // Attempt to parse year and month from the filename
    if not TryParseFileName(FileName, parsedPath, parsedSymbol, parsedYear, parsedMonth) then
        exit;

    if Assigned(IsMemoryLow) then
    begin
        if IsMemoryLow then
        begin
            var fL := FCachedFiles.Values.ToArray;
            TArray.Sort<TCachedFile>(
                fL,
                TComparer<TCachedFile>
                    .Construct(function(const Left, Right: TCachedFile): Integer begin Result := Sign(Left.LastUsed - Right.LastUsed); end)
            );

            for var i := 0 to High(fL) do
            begin
                FCachedFiles.Remove(fL[i].Name);
                if not IsMemoryLow then
                    break;
            end;
        end;

        var cachedFile: TCachedFile;
        if FCachedFiles.TryGetValue(Filename, cachedFile) then
        begin
            FCachedFiles.Remove(FileName);

            var age: TDateTime;
            if FileAge(Filename, age, true) then
            begin
                if age = cachedFile.Age then
                begin
                    cachedFile.LastUsed := Now;
                    FCachedFiles.Add(Filename, cachedFile);
                    exit(cachedFile.Data);
                end;
            end;
        end;
    end;

    if FileName.EndsWith('_zip', True) then
    begin
        var zipFileBytes := TFile.ReadAllBytes(FileName); // Read all bytes of the .zip file
        if Length(zipFileBytes) = 0 then
            exit;

        var capFileName := FileName;
        Result :=
            TFuture<TBytes>
                .Construct(
                    TLatch.Enqueue(LoadGate),
                    function: TBytes
                    begin
                        Result := TFile.ReadAllBytes(capFileName); // Read all bytes of the .zip file
                    end)
                .Chain<TArray<TDataPoint>>(
                    function(bytes: TBytes): TArray<TDataPoint>
                    begin
                        var zipMemoryStream := TBytesStream.Create(zipFileBytes); // Create a memory stream to hold the zip file content
                        try
                            zipMemoryStream.Position := 0;
                            Result := ReadCompressedData(zipMemoryStream);
                        finally
                            zipMemoryStream.Free;
                        end;
                    end);
    end
    else if TFile.Exists(FileName) then
    begin
        var capFileName := FileName;
        Result :=
            TFuture<TArray<TDataPoint>>.Construct(
                function: TArray<TDataPoint>
                begin
                    if TFile.Exists(capFileName) then
                    begin
                        var stream := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
                        try
                            try
                                Result := ReadUncompressedData(stream);
                            except
                                on E: EReadError do
                                begin
                                    raise EReadError.CreateFmt('File %s: %s', [capFileName, E.Message]);
                                end;
                            end;
                        finally
                            stream.Free;
                        end;
                    end;
                end
            );
    end;

    if Assigned(IsMemoryLow) then
    begin
        var cachedFile: TCachedFile;
        cachedFile.Name := Filename;
        var age: TDateTime;
        if FileAge(Filename, cachedFile.Age, true) then
        begin
            cachedFile.Data := Result;
            cachedFile.LastUsed := Now;
            FCachedFiles.Add(Filename, cachedFile);
        end;
    end;
end;

class function TDataSeries<T>.LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint>>;
// Local type declaration for storing file information for phase 1 processing
type
    TPhase1FileEntry = record
        Path: string;
        Year: Integer;
        Month: Integer;
    end;
var
    initialYear, initialMonth: Integer;
    currentTabYear, currentTabMonth: Integer;
    pathName, symbolName: string;
    tabFileEntryRecord: TPhase1FileEntry; // Variable to construct record instances
    loadedState: TState;
    loadedFiles: TArray<TFuture<TArray<TDataPoint>>>;
    liveData: TFuture<TArray<TDataPoint>>;

begin // Start of LoadDataSeries
    if not TryParseFileName(FileName, pathName, symbolName, initialYear, initialMonth) then
    begin
        raise EArgumentException.CreateFmt(
            'Invalid initial file name format: %s. Expected Symbol_Year_MM.Extension or Symbol_Year_MM.Extension_zip',
            [FileName]);
    end;

    currentTabYear := initialYear;
    currentTabMonth := initialMonth;
    var tabBaseNameFormat: string;
    var compressedTabPath, uncompressedTabPath: string;
    var actualTabFileToLoad: string;
    var maxTabYearFound := -1;
    var maxTabMonthFound := -1;

    var filesToLoadPhase1 := TList<TPhase1FileEntry>.Create; // Create list with the named record type
    try
        while True do
        begin
            tabBaseNameFormat := Format('%s_%.4d_%.2d.tab', [symbolName, currentTabYear, currentTabMonth]);
            compressedTabPath := TPath.Combine(pathName, tabBaseNameFormat + '_zip');
            uncompressedTabPath := TPath.Combine(pathName, tabBaseNameFormat);

            actualTabFileToLoad := '';
            if TFile.Exists(compressedTabPath) then
            begin
                actualTabFileToLoad := compressedTabPath;
            end
            else if TFile.Exists(uncompressedTabPath) then
            begin
                actualTabFileToLoad := uncompressedTabPath;
            end;

            if actualTabFileToLoad = '' then
                break; // No more sequential .tab files

            // Construct the record and add it to the list
            tabFileEntryRecord.Path := actualTabFileToLoad;
            tabFileEntryRecord.Year := currentTabYear;
            tabFileEntryRecord.Month := currentTabMonth;
            filesToLoadPhase1.Add(tabFileEntryRecord);

            // Advance to the next month
            Inc(currentTabMonth);
            if currentTabMonth > 12 then
            begin
                currentTabMonth := 1;
                Inc(currentTabYear);
            end;
        end;

        // Now load and process identified .tab files
        var loadedFileList := TList<TFuture<TArray<TDataPoint>>>.Create;
        var loadStates := TList<TState>.Create;
        try
            var LoadGate: TLatch;

            for var tabFileEntry: TPhase1FileEntry in filesToLoadPhase1 do // Iterate with the correct type
            begin
                var data := LoadDataFile(LoadGate, tabFileEntry.Path);

                loadedFileList.Add(TFuture<TArray<TDataPoint>>.Create(data));
                loadStates.Add(data.Done);

                if tabFileEntry.Year > maxTabYearFound then
                begin
                    maxTabYearFound := tabFileEntry.Year;
                    maxTabMonthFound := tabFileEntry.Month;
                end
                else if (tabFileEntry.Year = maxTabYearFound) and (tabFileEntry.Month > maxTabMonthFound) then
                begin
                    maxTabMonthFound := tabFileEntry.Month;
                end;
            end;

            var liveFileBaseName := Format('%s_%d_%02d.tab-live', [symbolName, maxTabYearFound, maxTabMonthFound]);
            liveData := LoadDataFile(LoadGate, TPath.Combine(pathName, liveFileBaseName));
            loadStates.Add(liveData.Done);
            loadedFileList.Add(liveData);

            loadedState := TState.All(loadStates.ToArray);
            loadedFiles := loadedFileList.ToArray;
        finally
            loadStates.Free;
            loadedFileList.Free;
        end;
    finally
        filesToLoadPhase1.Free;
    end;

    Result :=
        TFuture<TArray<TDataPoint>>.Construct(
            loadedState,
            function: TArray<TDataPoint>
            begin
                var tickList := TList<TDataPoint>.Create;
                try
                    var totalTicks := Length(liveData.Result);
                    var overallLastTabTickTime := 0.0;

                    for var P in loadedFiles do
                        Inc(totalTicks, Length(P.Result));

                    tickList.Capacity := totalTicks;

                    for var P in loadedFiles do
                    begin
                        if Length(P.Result) > 0 then
                        begin
                            for var iterTickData in P.Result do
                                if iterTickData.OADateTime > overallLastTabTickTime then
                                begin
                                    tickList.Add(iterTickData);
                                    overallLastTabTickTime := iterTickData.OADateTime;
                                end;
                        end;
                    end;

                    Result := tickList.ToArray;
                finally
                    tickList.Free;
                end;
            end
        );
end;

class function TDataSeries<T>.ReadCompressedData(const InputStream: TStream): TArray<TDataPoint>;
var
    decompressionStream: TStream; // Holds uncompressed data of a single zip entry
    localHeader: TZipHeader;
    entryIndex: Integer;
    i: Integer;
    zipFileInstance: TZipFile;
begin
    // Initialize Result record
    SetLength(Result, 0);

    decompressionStream := nil;
    zipFileInstance := nil;

    try
        InputStream.Position := 0; // Reset stream position to the beginning for TZipFile

        // Open TZipFile from this memory stream
        zipFileInstance := TZipFile.Create;
        zipFileInstance.Open(InputStream, TZipMode.zmRead); // Open the zip from the memory stream

        if zipFileInstance.FileCount = 0 then
        begin
            // If TZipFile opens an empty or invalid stream successfully but finds no files
            exit; // FileSuccessfullyLoaded remains False
        end;

        entryIndex := -1;
        for i := 0 to zipFileInstance.FileCount - 1 do
        begin
            if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
            begin
                entryIndex := i;
                break;
            end;
        end;

        if entryIndex = -1 then // If specific name not found, default to the first entry
        begin
            if zipFileInstance.FileCount > 0 then
                entryIndex := 0
            else
                exit; // Should not happen if FileCount > 0 check passed
        end;

        // Decompress the specific entry from the in-memory zip file.
        // TZipFile.Read creates and populates decompressionStream with uncompressed data.
        zipFileInstance.Read(entryIndex, decompressionStream, localHeader);

        if not Assigned(decompressionStream) then
            exit; // Exit if no valid stream could be prepared

        Result := ReadUncompressedData(decompressionStream);
    finally
        // Ensure all potentially created objects are freed
        if Assigned(decompressionStream) then // Holds uncompressed data from a zip entry
            decompressionStream.Free;
        if Assigned(zipFileInstance) then
            zipFileInstance.Free;
    end;
end;

class function TDataSeries<T>.ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint>;
var
    fileSize: Int64;
    recordCount: Integer;
    bytesRead: Integer;
begin
    // Initialize Result record
    SetLength(Result, 0);

    InputStream.Position := 0;

    fileSize := InputStream.Size;
    if fileSize = 0 then
        exit;

    if (fileSize mod SizeOf(TDataPoint)) <> 0 then
        raise EReadError.CreateFmt('File corrupted. Size %d is not a multiple of record size %d.', [fileSize, SizeOf(TDataPoint)]);

    recordCount := fileSize div SizeOf(TDataPoint);
    if recordCount > 0 then
    begin
        SetLength(Result, recordCount);
        bytesRead := InputStream.Read(Result[0], fileSize);
        if bytesRead <> fileSize then
            raise EReadError.CreateFmt('Read error. Expected to read %d bytes, but actually read %d bytes.', [fileSize, bytesRead]);
    end;
end;

end.