MycLib/Src/Myc.Trade.DataStream.pas

unit Myc.Trade.DataStream;

interface

uses
    System.SysUtils,
    System.Classes,
    System.Generics.Collections,
    System.Generics.Defaults,
    System.IOUtils,
    Myc.Futures,
    Myc.Signals,
    Myc.Mutable,
    Myc.Trade.Types,
    Myc.Data.Pipeline,
    Myc.Core.FileCache;

type
    // Match C# M1Record (Pack = 1, 48 Bytes)
    TM1Record = packed record
        Time: Double;
        Open: Double;
        High: Double;
        Low: Double;
        Close: Double;
        Spread: Single;
        Volume: Integer;
    end;

    // Match C# TickRecord (Pack = 1, 24 Bytes)
    TTickRecord = packed record
        Time: Double;
        Ask: Double;
        Bid: Double;
    end;

    IDataServer = interface
        procedure ClearCache;
        function EnumerateSymbols: TFuture<TArray<String>>;
    end;

    // Interface for an instantiable data server.
    IDataServer<T: record> = interface(IDataServer)
        function ProcessData(const Symbol: String; const Terminated: TState; const Processor: IConsumer<TArray<TDataPoint<T>>>): TState;
    end;

    // Metadata for a monthly data file (SYMBOL_YYYY_MM.EXT)
    TDataFile = record
    private
        FPath: String;
        FSymbol: String; // Includes extension, e.g. "EURUSD.M1"
        FYear: Integer;
        FMonth: Integer;
        FBasename: String;
        function GetIsValid: Boolean;
    public
        constructor Create(const APath, ASymbol: String; AYear, AMonth: Integer; const ABasename: String);
        function GetBaseFileName: string;
        function GetFullFileName: string;
        property IsValid: Boolean read GetIsValid;
        property Path: String read FPath;
        property Symbol: String read FSymbol;
        property Year: Integer read FYear;
        property Month: Integer read FMonth;
    end;

    // Generic server base handling the pipeline and preloading.
    TDataServer<T: record; R: record> = class(TInterfacedObject, IDataServer, IDataServer<T>)
    private
        // Cache stores binary blobs of the MAPPED data (TDataPoint<T>)
        FCachedFiles: IDataFileCache<TArray<TBytes>>;
        FPath: String;
        FSymbols: TFuture<TDictionary<String, TArray<TDataFile>>>;
        function GetPath: String;

        function ProcessChunks(const MappedBlobs: TArray<TBytes>; Terminated: TState; Processor: IConsumer<TArray<TDataPoint<T>>>): TState;

        function ProcessFile(
            FileInfo: TDataFile;
            const DataFile: TFuture<TArray<TBytes>>;
            const Terminated: TState;
            Processor: IConsumer<TArray<TDataPoint<T>>>
        ): TState;

    strict private
        class var
            FLoadGate: TLatch;
        class constructor CreateClass;

    protected
        // Maps the raw file record to the internal TDataPoint<T>
        function MapRecord(const RecordBuffer: R): TDataPoint<T>; virtual; abstract;

        // Returns Future of Processed Blobs (Serialized TDataPoint<T>)
        function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TBytes>>;

        function GetNextFile(const Curr: TDataFile): TDataFile; virtual;
        function ReadCompressedData(const InputStream: TStream): TArray<TBytes>;
        function ReadUncompressedData(const InputStream: TStream): TArray<TBytes>;

    public
        // Cache is injected. It stores TArray<TBytes>.
        constructor Create(const ACache: IDataFileCache<TArray<TBytes>>; const APath: String);
        destructor Destroy; override;
        procedure AfterConstruction; override;
        procedure ClearCache;
        procedure UpdateSymbols;
        function FindFirstFile(const Symbol: string): TFuture<TDataFile>;
        function ParseFileName(const FileName: string): TDataFile;
        function EnumerateSymbols: TFuture<TArray<String>>;

        function LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TBytes>>;
        function ProcessData(const Symbol: String; const Terminated: TState; const Processor: IConsumer<TArray<TDataPoint<T>>>): TState;

        property Path: String read GetPath;
    end;

    ITickDataServer = interface(IDataServer<TTickRecord>)
        ['{A6DE8B82-B0FB-48DB-A39A-CE91D861C5D2}']
    end;

    TTickFileServer = class(TDataServer<TTickRecord, TTickRecord>, ITickDataServer)
    protected
        function MapRecord(const RecordBuffer: TTickRecord): TDataPoint<TTickRecord>; override;
    end;

    IM1DataServer = interface(IDataServer<TOhlcItem>)
        ['{7BB980D9-1969-49A4-991F-908F2DDAC1A0}']
    end;

    TM1FileServer = class(TDataServer<TOhlcItem, TM1Record>, IM1DataServer)
    protected
        function MapRecord(const RecordBuffer: TM1Record): TDataPoint<TOhlcItem>; override;
    end;

implementation

uses
    System.Zip,
    System.ZLib,
    System.Math,
    System.StrUtils,
    Myc.TaskManager;

const
    ChunkSize = 1024;

{ TDataFile }

constructor TDataFile.Create(const APath, ASymbol: String; AYear, AMonth: Integer; const ABasename: String);
begin
    FPath := APath;
    FSymbol := ASymbol;
    FYear := AYear;
    FMonth := AMonth;
    FBasename := ABasename;
end;

function TDataFile.GetBaseFileName: string;
begin
    Result := FBasename;
end;

function TDataFile.GetFullFileName: string;
var
    ext: string;
begin
    ext := TPath.GetExtension(FSymbol);
    Result := TPath.Combine(FPath, FBasename + ext);
end;

function TDataFile.GetIsValid: Boolean;
begin
    Result := (FSymbol <> '') and (FYear > 0) and (FMonth >= 1) and (FMonth <= 12);
end;

{ TDataServer<T, R> }

class constructor TDataServer<T, R>.CreateClass;
begin
    FLoadGate := TLatch.CreateLatch(0);
end;

constructor TDataServer<T, R>.Create(const ACache: IDataFileCache<TArray<TBytes>>; const APath: String);
begin
    inherited Create;
    FPath := APath;
    FCachedFiles := ACache;
end;

destructor TDataServer<T, R>.Destroy;
begin
    inherited Destroy;
end;

procedure TDataServer<T, R>.AfterConstruction;
begin
    inherited;
    UpdateSymbols;
end;

procedure TDataServer<T, R>.ClearCache;
begin
    if Assigned(FCachedFiles) then
        FCachedFiles.Clear;
end;

procedure TDataServer<T, R>.UpdateSymbols;
begin
    FSymbols :=
        TFuture<TDictionary<String, TArray<TDataFile>>>.Construct(
            FSymbols.Done,
            function: TDictionary<String, TArray<TDataFile>>
            var
                fileNames: TArray<string>;
                tempSymbolMap: TDictionary<String, TList<TDataFile>>;
                dataFile: TDataFile;
                symbolList: TList<TDataFile>;
            begin
                Result := TDictionary<String, TArray<TDataFile>>.Create;
                tempSymbolMap := TDictionary<String, TList<TDataFile>>.Create;
                try
                    if not TDirectory.Exists(FPath) then
                        exit;

                    fileNames := TDirectory.GetFiles(FPath);
                    for var currentFile in fileNames do
                    begin
                        dataFile := ParseFileName(currentFile);
                        if dataFile.IsValid then
                        begin
                            if not tempSymbolMap.TryGetValue(dataFile.Symbol, symbolList) then
                            begin
                                symbolList := TList<TDataFile>.Create;
                                tempSymbolMap.Add(dataFile.Symbol, symbolList);
                            end;
                            symbolList.Add(dataFile);
                        end;
                    end;

                    for var kvp in tempSymbolMap do
                    begin
                        symbolList := kvp.Value;
                        symbolList.Sort(
                            TComparer<TDataFile>.Construct(
                                function(const Left, Right: TDataFile): Integer
                                begin
                                    if (Left.Year <> Right.Year) then
                                        Result := Left.Year - Right.Year
                                    else
                                        Result := Left.Month - Right.Month;
                                end
                            )
                        );
                        Result.Add(kvp.Key, symbolList.ToArray);
                    end;
                finally
                    for var kvp in tempSymbolMap do
                        kvp.Value.Free;
                    tempSymbolMap.Free;
                end;
            end
        );

    FSymbols.Manage;
end;

function TDataServer<T, R>.EnumerateSymbols: TFuture<TArray<String>>;
begin
    Result :=
        FSymbols.Chain<TArray<String>>(
            function(const Symbols: TDictionary<String, TArray<TDataFile>>): TArray<String> begin Result := Symbols.Keys.ToArray; end
        );
end;

function TDataServer<T, R>.FindFirstFile(const Symbol: string): TFuture<TDataFile>;
begin
    var symName := Symbol;
    Result :=
        FSymbols.Chain<TDataFile>(
            function(const Symbols: TDictionary<String, TArray<TDataFile>>): TDataFile
            var
                symbolFiles: TArray<TDataFile>;
            begin
                if (Symbols.TryGetValue(symName, symbolFiles)) and (Length(symbolFiles) > 0) then
                    Result := symbolFiles[0]
                else
                    Result := Default(TDataFile);
            end
        );
end;

function TDataServer<T, R>.GetNextFile(const Curr: TDataFile): TDataFile;
var
    allSymbolFiles: TArray<TDataFile>;
    foundIndex: Integer;
begin
    Result := Default(TDataFile);
    if (not Curr.IsValid) or (not FSymbols.Done.IsSet) then
        exit;

    if not FSymbols.Value.TryGetValue(Curr.Symbol, allSymbolFiles) then
        exit;

    foundIndex := -1;
    for var i := 0 to High(allSymbolFiles) do
    begin
        if (allSymbolFiles[i].GetBaseFileName = Curr.GetBaseFileName) then
        begin
            foundIndex := i;
            break;
        end;
    end;

    if (foundIndex <> -1) and (foundIndex < High(allSymbolFiles)) then
        Result := allSymbolFiles[foundIndex + 1];
end;

function TDataServer<T, R>.GetPath: String;
begin
    Result := FPath;
end;

function TDataServer<T, R>.ParseFileName(const FileName: string): TDataFile;
var
    fileNameNoPath, baseName, symbol, ext: string;
    parts: TArray<string>;
    year, month: Integer;
begin
    Result := Default(TDataFile);
    fileNameNoPath := TPath.GetFileName(FileName);
    ext := TPath.GetExtension(fileNameNoPath);
    baseName := TPath.GetFileNameWithoutExtension(fileNameNoPath);

    parts := baseName.Split(['_']);
    if (Length(parts) < 3) then
        exit;

    if (not TryStrToInt(parts[High(parts)], month)) or (not TryStrToInt(parts[High(parts) - 1], year)) then
        exit;

    symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2)) + ext;
    Result := TDataFile.Create(TPath.GetDirectoryName(FileName), symbol, year, month, baseName);
end;

function TDataServer<T, R>.LoadDataFile(const DataFile: TDataFile): TFuture<TArray<TBytes>>;
begin
    if (DataFile.IsValid) then
        Result :=
            FCachedFiles.GetOrAdd(
                DataFile.GetFullFileName,
                function(const Filename: String): TFuture<TArray<TBytes>> begin Result := DoLoad(Filename, FLoadGate); end
            )
    else
        Result := TFuture<TArray<TBytes>>.Construct(TArray<TBytes>(nil));
end;

function TDataServer<T, R>.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TBytes>>;
begin
    var capFileName := FileName;
    Result :=
        TFuture<TBytes>
            .Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
            .Chain<TArray<TBytes>>(
                function(Bytes: TBytes): TArray<TBytes>
                var
                    compStream: TBytesStream;
                begin
                    compStream := TBytesStream.Create(Bytes);
                    try
                        Result := ReadCompressedData(compStream);
                    finally
                        compStream.Free;
                    end;
                end);
end;

function TDataServer<T, R>.ReadCompressedData(const InputStream: TStream): TArray<TBytes>;
var
    zip: TZipFile;
    decompStream: TStream;
    header: TZipHeader;
    entryIdx: Integer;
begin
    Result := nil;
    InputStream.Position := 0;
    zip := TZipFile.Create;
    try
        zip.Open(InputStream, TZipMode.zmRead);
        entryIdx := -1;
        // Search for the .bin entry created by C#
        for var i := 0 to zip.FileCount - 1 do
        begin
            if (zip.FileNames[i].EndsWith('.bin', true)) then
            begin
                entryIdx := i;
                break;
            end;
        end;

        if (entryIdx = -1) then
            exit;

        zip.Read(entryIdx, decompStream, header);
        try
            Result := ReadUncompressedData(decompStream);
        finally
            decompStream.Free;
        end;
    finally
        zip.Free;
    end;
end;

function TDataServer<T, R>.ReadUncompressedData(const InputStream: TStream): TArray<TBytes>;
var
    totalRecords, chunkCount, i, j, recsInChunk: Integer;
    inputRecordSize, outputPointSize: Integer;
    chunkBuffer: TArray<TDataPoint<T>>;
    rawBufferChunk: TArray<R>; // Only buffer one chunk at a time
begin
    Result := nil;
    inputRecordSize := SizeOf(R);
    outputPointSize := SizeOf(TDataPoint<T>);

    // Ensure we are working with complete records
    if (InputStream.Size mod inputRecordSize <> 0) then
        raise Exception.Create('Stream size is not a multiple of record size.');

    totalRecords := InputStream.Size div inputRecordSize;
    if (totalRecords = 0) then
        exit;

    chunkCount := (totalRecords + ChunkSize - 1) div ChunkSize;
    SetLength(Result, chunkCount);
    SetLength(rawBufferChunk, ChunkSize); // Reuse buffer

    InputStream.Position := 0;

    for i := 0 to chunkCount - 1 do
    begin
        recsInChunk := Min(ChunkSize, totalRecords - (i * ChunkSize));

        // Read only needed bytes for this chunk
        InputStream.ReadBuffer(rawBufferChunk[0], recsInChunk * inputRecordSize);

        // Temp buffer for mapped objects
        SetLength(chunkBuffer, recsInChunk);

        for j := 0 to recsInChunk - 1 do
        begin
            chunkBuffer[j] := MapRecord(rawBufferChunk[j]);
        end;

        // Serialize mapped data to TBytes
        SetLength(Result[i], recsInChunk * outputPointSize);
        Move(chunkBuffer[0], Result[i][0], recsInChunk * outputPointSize);
    end;
end;

function TDataServer<T, R>.ProcessChunks(
    const MappedBlobs: TArray<TBytes>;
    Terminated: TState;
    Processor: IConsumer<TArray<TDataPoint<T>>>
): TState;
begin
    var callProcess :=
        function(Idx: Integer): TFunc<TState>
        begin
            // Capture the blob
            var cBlob := MappedBlobs[Idx];
            Result :=
                function: TState
                var
                    dataChunk: TArray<TDataPoint<T>>;
                    count: Integer;
                begin
                    if not Terminated.IsSet then
                    begin
                        // Quick deserialize (Mem Copy)
                        count := Length(cBlob) div SizeOf(TDataPoint<T>);
                        SetLength(dataChunk, count);
                        if count > 0 then
                            Move(cBlob[0], dataChunk[0], Length(cBlob));

                        Result := Processor.Consume(dataChunk);
                    end;
                end;
        end;

    for var i := 0 to High(MappedBlobs) do
    begin
        if not Terminated.IsSet then
        begin
            var q := TaskManager.RunTask(Result, callProcess(i));
            Result := q;
        end;
    end;
end;

function TDataServer<T, R>.ProcessFile(
    FileInfo: TDataFile;
    const DataFile: TFuture<TArray<TBytes>>;
    const Terminated: TState;
    Processor: IConsumer<TArray<TDataPoint<T>>>
): TState;
begin
    if (not FileInfo.IsValid) or (Terminated.IsSet) then
        exit(DataFile.Done);

    var nextFileInfo := GetNextFile(FileInfo);
    var nextFile := LoadDataFile(nextFileInfo);
    var cTerminated := Terminated;

    Result :=
        DataFile.Chain(
            function(const MappedBlobs: TArray<TBytes>): TState
            begin
                var done := ProcessChunks(MappedBlobs, cTerminated, Processor);
                Result :=
                    TaskManager
                        .RunTask(done, function: TState begin Result := ProcessFile(nextFileInfo, nextFile, cTerminated, Processor); end);
            end
        );
end;

function TDataServer<T, R>.ProcessData(
    const Symbol: String;
    const Terminated: TState;
    const Processor: IConsumer<TArray<TDataPoint<T>>>
): TState;
begin
    var cProc := Processor;
    var cTerm := Terminated;
    Result :=
        FindFirstFile(Symbol)
            .Chain(function(const Info: TDataFile): TState begin Result := ProcessFile(Info, LoadDataFile(Info), cTerm, cProc); end);
end;

{ TTickFileServer }

function TTickFileServer.MapRecord(const RecordBuffer: TTickRecord): TDataPoint<TTickRecord>;
begin
    Result := TDataPoint<TTickRecord>.Create(RecordBuffer.Time, RecordBuffer);
end;

{ TM1FileServer }

function TM1FileServer.MapRecord(const RecordBuffer: TM1Record): TDataPoint<TOhlcItem>;
begin
    Result :=
        TDataPoint<TOhlcItem>.Create(
            RecordBuffer.Time,
            TOhlcItem.Create(RecordBuffer.Open, RecordBuffer.High, RecordBuffer.Low, RecordBuffer.Close, RecordBuffer.Volume)
        );
end;

end.