Removed bottleneck in DataStream

Src/Myc.Trade.DataStream.pas

@@ -34,13 +34,12 @@ type
         FSymbol: String;
         FYear: Integer;
         FMonth: Integer;
+        FBasename: String;
         function GetIsValid: Boolean;
     public
-        constructor Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer);
+        constructor Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer; const ABasename: String);
         function GetBaseFileName: string;
         function GetFullFileName: string;
-        // Gets the next consecutive data file, if it exists on disk.
-        function GetNextFile: TAuraDataFile;
         property IsValid: Boolean read GetIsValid;
         property Extension: String read FExtension;
         property Path: String read FPath;
@@ -54,12 +53,10 @@ type
     private
         FCachedFiles: TDataFileCache<TArray<TArray<TDataPoint<T>>>>;
         FPath: String;
-        FSymbols: TFuture<TDictionary<String, TAuraDataFile>>;
+        // The cache now holds a sorted array of all files for each symbol.
+        FSymbols: TFuture<TDictionary<String, TArray<TAuraDataFile>>>;
         function GetPath: String;
 
-        // Used by cache to actually load an uncached file.
-        function DoLoad(const FileName: string): TFuture<TArray<TArray<TDataPoint<T>>>>;
-
         function ProcessChunks(
             const DataChunks: TArray<TArray<TDataPoint<T>>>;
             Terminated: TState;
@@ -78,10 +75,10 @@ type
             FLoadGate: TLatch;
 
     protected
-        // Read data from stream and return it chunked.
-        class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<T>>>; virtual; abstract;
-        // Read data from stream and return it chunked.
-        class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<T>>>; virtual; abstract;
+        // Used by cache to actually load an uncached file. Now virtual and abstract.
+        function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<T>>>>; virtual; abstract;
+        // Gets the next consecutive data file from the cached, sorted list.
+        function GetNextFile(const Curr: TAuraDataFile): TAuraDataFile; virtual;
 
     public
         constructor Create(const APath: String);
@@ -110,12 +107,36 @@ type
     end;
 
     TAuraTABFileServer = class(TAuraDataServer<TAuraAskBidFileItem>)
+    private
+        class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>;
+        class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>;
     protected
-        // Scans a directory and returns the oldest file found for each symbol.
-        class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>; override;
-        class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>; override;
+        function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>; override;
+    public
+        function ParseFileName(const FileName: string): TAuraDataFile; override;
+    end;
+
+    // File record for cTrader M1 data
+    TAuraM1FileItem = packed record
+        OADateTime: Double;
+        Open: Int64;
+        High: Int64;
+        Low: Int64;
+        Close: Int64;
+        Spread: Int64;
+        TickVolume: Integer;
+        Digits: Integer;
+    end;
+
+    // File server for cTrader M1 data. Note that the generic type is TOhlcItem,
+    // as the server converts the file data into standard OHLC items.
+    TAuraM1FileServer = class(TAuraDataServer<TOhlcItem>)
+    private
+        class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
+        class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
+    protected
+        function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>; override;
     public
-        function LoadDataSeries(const InitialFile: TAuraDataFile): TFuture<TArray<TDataPoint<TAuraAskBidFileItem>>>;
         function ParseFileName(const FileName: string): TAuraDataFile; override;
     end;
 
@@ -123,6 +144,7 @@ implementation
 
 uses
     System.Zip,
+    System.ZLib,
     System.Math,
     System.StrUtils,
     Myc.TaskManager;
@@ -133,18 +155,19 @@ const
 
 { TAuraDataFile }
 
-constructor TAuraDataFile.Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer);
+constructor TAuraDataFile.Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer; const ABasename: String);
 begin
     FPath := APath;
     FSymbol := ASymbol;
     FExtension := AExtension;
     FYear := AYear;
     FMonth := AMonth;
+    FBasename := ABasename;
 end;
 
 function TAuraDataFile.GetBaseFileName: string;
 begin
-    Result := Format('%s_%.4d_%.2d', [FSymbol, FYear, FMonth]);
+    Result := FBasename;
 end;
 
 function TAuraDataFile.GetFullFileName: string;
@@ -157,36 +180,6 @@ begin
     Result := (FSymbol <> '') and (FYear > 0);
 end;
 
-function TAuraDataFile.GetNextFile: TAuraDataFile;
-var
-    nextMonth, nextYear: Integer;
-    nextFile: TAuraDataFile;
-begin
-    if not IsValid then
-        exit(Default(TAuraDataFile));
-
-    nextMonth := FMonth + 1;
-    nextYear := FYear;
-    if (nextMonth > 12) then
-    begin
-        nextMonth := 1;
-        Inc(nextYear);
-    end;
-
-    // Probe for zipped file first
-    nextFile := TAuraDataFile.Create(FPath, FSymbol, '.tab_zip', nextYear, nextMonth);
-    if TFile.Exists(nextFile.GetFullFileName) then
-        exit(nextFile);
-
-    // Probe for uncompressed file
-    nextFile := TAuraDataFile.Create(FPath, FSymbol, '.tab', nextYear, nextMonth);
-    if TFile.Exists(nextFile.GetFullFileName) then
-        exit(nextFile);
-
-    // No next file found
-    Result := Default(TAuraDataFile);
-end;
-
 { TAuraDataServer<T> }
 
 constructor TAuraDataServer<T>.Create(const APath: String);
@@ -195,8 +188,13 @@ begin
     FPath := APath;
 
     FCachedFiles :=
-        TDataFileCache<TArray<TArray<TDataPoint<T>>>>
-            .Create(function(const Filename: String): TFuture<TArray<TArray<TDataPoint<T>>>> begin Result := DoLoad(Filename); end);
+        TDataFileCache<TArray<TArray<TDataPoint<T>>>>.Create(
+            function(const Filename: String): TFuture<TArray<TArray<TDataPoint<T>>>>
+            begin
+                // Pass the private load gate to the virtual DoLoad method.
+                Result := DoLoad(Filename, FLoadGate);
+            end
+        );
 end;
 
 destructor TAuraDataServer<T>.Destroy;
@@ -217,92 +215,61 @@ begin
     FCachedFiles.Clear;
 end;
 
-function TAuraDataServer<T>.DoLoad(const FileName: string): TFuture<TArray<TArray<TDataPoint<T>>>>;
-begin
-    Result := TFuture<TArray<TArray<TDataPoint<T>>>>.Null;
-
-    if not TFile.Exists(FileName) then
-        exit;
-
-    var capFileName := FileName;
-    if FileName.EndsWith('_zip', True) then
-    begin
-        Result :=
-            TFuture<TBytes>
-                .Construct(FLoadGate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
-                .Chain<TArray<TArray<TDataPoint<T>>>>(
-                    function(bytes: TBytes): TArray<TArray<TDataPoint<T>>>
-                    begin
-                        var zipMemoryStream := TBytesStream.Create(bytes);
-                        try
-                            zipMemoryStream.Position := 0;
-                            Result := ReadCompressedData(zipMemoryStream);
-                        finally
-                            zipMemoryStream.Free;
-                        end;
-                    end);
-    end
-    else
-    begin
-        Result :=
-            TFuture<TArray<TArray<TDataPoint<T>>>>.Construct(
-                FLoadGate.Enqueue,
-                function: TArray<TArray<TDataPoint<T>>>
-                begin
-                    if TFile.Exists(capFileName) then
-                    begin
-                        var stream := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
-                        try
-                            try
-                                Result := ReadUncompressedData(stream);
-                            except
-                                on E: EReadError do
-                                    raise EReadError.CreateFmt('File %s: %s', [capFileName, E.Message]);
-                            end;
-                        finally
-                            stream.Free;
-                        end;
-                    end;
-                end
-            );
-    end;
-end;
-
 procedure TAuraDataServer<T>.UpdateSymbols;
 begin
     FSymbols :=
-        TFuture<TDictionary<String, TAuraDataFile>>.Construct(
+        TFuture<TDictionary<String, TArray<TAuraDataFile>>>.Construct(
             FSymbols.Done,
-            function: TDictionary<String, TAuraDataFile>
+            function: TDictionary<String, TArray<TAuraDataFile>>
             var
                 fileNames: TArray<string>;
-                currentFile: string;
+                tempSymbolMap: TDictionary<String, TList<TAuraDataFile>>;
                 dataFile: TAuraDataFile;
-                trackedInfo: TAuraDataFile;
+                symbolList: TList<TAuraDataFile>;
             begin
-                Result := TDictionary<string, TAuraDataFile>.Create;
-                if not TDirectory.Exists(FPath) then
-                    exit;
+                Result := TDictionary<String, TArray<TAuraDataFile>>.Create;
+                tempSymbolMap := TDictionary<String, TList<TAuraDataFile>>.Create;
+                try
+                    if not TDirectory.Exists(FPath) then
+                        exit;
 
-                fileNames := TDirectory.GetFiles(FPath);
-                for currentFile in fileNames do
-                begin
-                    dataFile := ParseFileName(currentFile);
-                    if dataFile.IsValid then
+                    fileNames := TDirectory.GetFiles(FPath);
+                    for var currentFile in fileNames do
                     begin
-                        if Result.TryGetValue(dataFile.Symbol, trackedInfo) then
+                        dataFile := ParseFileName(currentFile);
+                        if dataFile.IsValid then
                         begin
-                            if (dataFile.Year < trackedInfo.Year)
-                                or ((dataFile.Year = trackedInfo.Year) and (dataFile.Month < trackedInfo.Month)) then
+                            if not tempSymbolMap.TryGetValue(dataFile.Symbol, symbolList) then
                             begin
-                                Result.AddOrSetValue(dataFile.Symbol, dataFile);
+                                symbolList := TList<TAuraDataFile>.Create;
+                                tempSymbolMap.Add(dataFile.Symbol, symbolList);
                             end;
-                        end
-                        else
-                        begin
-                            Result.Add(dataFile.Symbol, dataFile);
+                            symbolList.Add(dataFile);
                         end;
                     end;
+
+                    for var kvp in tempSymbolMap do
+                    begin
+                        symbolList := kvp.Value;
+                        symbolList.Sort(
+                            TComparer<TAuraDataFile>.Construct(
+                                function(const Left, Right: TAuraDataFile): Integer
+                                begin
+                                    if Left.Year < Right.Year then
+                                        Result := -1
+                                    else if Left.Year > Right.Year then
+                                        Result := 1
+                                    else
+                                        Result := Left.Month - Right.Month;
+                                end
+                            )
+                        );
+                        Result.Add(kvp.Key, symbolList.ToArray);
+                    end;
+                finally
+                    for var kvp in tempSymbolMap do
+                        kvp.Value.Free;
+                    tempSymbolMap.Free;
                 end;
             end
         );
@@ -314,7 +281,7 @@ function TAuraDataServer<T>.EnumerateSymbols: TFuture<TArray<String>>;
 begin
     Result :=
         FSymbols.Chain<TArray<String>>(
-            function(Symbols: TDictionary<String, TAuraDataFile>): TArray<String> begin Result := Symbols.Keys.ToArray; end
+            function(const Symbols: TDictionary<String, TArray<TAuraDataFile>>): TArray<String> begin Result := Symbols.Keys.ToArray; end
         );
 end;
 
@@ -323,14 +290,46 @@ begin
     var symName := Symbol;
     Result :=
         FSymbols.Chain<TAuraDataFile>(
-            function(Symbols: TDictionary<String, TAuraDataFile>): TAuraDataFile
+            function(const Symbols: TDictionary<String, TArray<TAuraDataFile>>): TAuraDataFile
+            var
+                symbolFiles: TArray<TAuraDataFile>;
             begin
-                if not Symbols.TryGetValue(symName, Result) then
+                if Symbols.TryGetValue(symName, symbolFiles) and (Length(symbolFiles) > 0) then
+                    Result := symbolFiles[0]
+                else
                     Result := Default(TAuraDataFile);
             end
         );
 end;
 
+function TAuraDataServer<T>.GetNextFile(const Curr: TAuraDataFile): TAuraDataFile;
+var
+    allSymbolFiles: TArray<TAuraDataFile>;
+    foundIndex: Integer;
+begin
+    Result := Default(TAuraDataFile);
+    if not Curr.IsValid or not FSymbols.Done.IsSet then
+        exit;
+
+    if not FSymbols.Value.TryGetValue(Curr.Symbol, allSymbolFiles) then
+        exit;
+
+    foundIndex := -1;
+    for var i := 0 to High(allSymbolFiles) do
+    begin
+        if allSymbolFiles[i].GetBaseFileName() = Curr.GetBaseFileName() then
+        begin
+            foundIndex := i;
+            break;
+        end;
+    end;
+
+    if (foundIndex <> -1) and (foundIndex < High(allSymbolFiles)) then
+    begin
+        Result := allSymbolFiles[foundIndex + 1];
+    end;
+end;
+
 function TAuraDataServer<T>.GetPath: String;
 begin
     Result := FPath;
@@ -357,7 +356,9 @@ end;
 function TAuraDataServer<T>.LoadDataFile(const DataFile: TAuraDataFile): TFuture<TArray<TArray<TDataPoint<T>>>>;
 begin
     if DataFile.IsValid then
-        Result := FCachedFiles.GetOrAdd(DataFile.GetFullFileName);
+        Result := FCachedFiles.GetOrAdd(DataFile.GetFullFileName)
+    else
+        Result := TFuture<TArray<TArray<TDataPoint<T>>>>.Construct(TArray<TArray<TDataPoint<T>>>(nil));
 end;
 
 function TAuraDataServer<T>.ProcessChunks(
@@ -397,7 +398,7 @@ begin
         exit(DataFile.Done);
 
     // Read ahead the next file, while processing the current one
-    var nextFileInfo := FileInfo.GetNextFile;
+    var nextFileInfo := GetNextFile(FileInfo);
     var nextFile := LoadDataFile(nextFileInfo);
 
     var cTerminated := Terminated;
@@ -416,90 +417,52 @@ begin
         );
 end;
 
-function TAuraTABFileServer.LoadDataSeries(const InitialFile: TAuraDataFile): TFuture<TArray<TDataPoint<TAuraAskBidFileItem>>>;
-var
-    loadedState: TState;
-    loadedFiles: TArray<TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>>;
-    liveData: TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>;
-    tabFiles: TList<TAuraDataFile>;
-    liveFile: TAuraDataFile;
-    currentFileInfo: TAuraDataFile;
+{ TAuraTABFileServer }
+
+function TAuraTABFileServer.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>;
 begin
-    tabFiles := TList<TAuraDataFile>.Create;
-    try
-        currentFileInfo := InitialFile;
-        while currentFileInfo.IsValid do
-        begin
-            tabFiles.Add(currentFileInfo);
-            currentFileInfo := currentFileInfo.GetNextFile;
-        end;
+    Result := TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>.Null;
 
-        liveFile := Default(TAuraDataFile);
-        if tabFiles.Count > 0 then
-        begin
-            var lastTabFile := tabFiles.Last;
-            var liveFileBaseName := Format('%s_%d_%02d.tab-live', [lastTabFile.Symbol, lastTabFile.Year, lastTabFile.Month]);
-            var potentialLivePath := TPath.Combine(lastTabFile.Path, liveFileBaseName);
-            if TFile.Exists(potentialLivePath) then
-                liveFile := ParseFileName(potentialLivePath);
-        end;
+    if not TFile.Exists(FileName) then
+        exit;
 
-        var loadedFileList := TList<TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>>.Create;
-        var loadStates := TList<TState>.Create;
-        try
-            for var fileInfo in tabFiles do
-            begin
-                var data := LoadDataFile(fileInfo);
-                loadedFileList.Add(data);
-                loadStates.Add(data.Done);
-            end;
-
-            liveData := TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>.Null;
-            if liveFile.IsValid then
-            begin
-                liveData := LoadDataFile(liveFile);
-                loadedFileList.Add(liveData);
-                loadStates.Add(liveData.Done);
-            end;
-
-            loadedState := TState.All(loadStates.ToArray);
-            loadedFiles := loadedFileList.ToArray;
-        finally
-            loadStates.Free;
-            loadedFileList.Free;
-        end;
-    finally
-        tabFiles.Free;
+    var capFileName := FileName;
+    if FileName.EndsWith('_zip', True) then
+    begin
+        Result :=
+            TFuture<TBytes>
+                .Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
+                .Chain<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>(
+                    function(bytes: TBytes): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>
+                    begin
+                        var compMemoryStream := TBytesStream.Create(bytes);
+                        try
+                            compMemoryStream.Position := 0;
+                            Result := ReadCompressedData(compMemoryStream);
+                        finally
+                            compMemoryStream.Free;
+                        end;
+                    end);
+    end
+    else
+    begin
+        Result :=
+            TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>.Construct(
+                Gate.Enqueue,
+                function: TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>
+                begin
+                    if TFile.Exists(capFileName) then
+                    begin
+                        var stream := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
+                        try
+                            Result := ReadUncompressedData(stream);
+                        finally
+                            stream.Free;
+                        end;
+                    end;
+                end
+            );
     end;
-
-    Result :=
-        TFuture<TArray<TDataPoint<TAuraAskBidFileItem>>>.Construct(
-            loadedState,
-            function: TArray<TDataPoint<TAuraAskBidFileItem>>
-            begin
-                var tickList := TList<TDataPoint<TAuraAskBidFileItem>>.Create;
-                try
-                    var overallLastTabTickTime: TDateTime := 0;
-                    var cnt := 0;
-                    for var P in loadedFiles do
-                        for var chunk in P.Value do
-                            inc(cnt, Length(chunk));
-                    tickList.Capacity := cnt;
-
-                    for var P in loadedFiles do
-                        for var chunk in P.Value do
-                            for var iterTickData in chunk do
-                                if overallLastTabTickTime < iterTickData.Time then
-                                begin
-                                    tickList.Add(iterTickData);
-                                    overallLastTabTickTime := iterTickData.Time;
-                                end;
-                    Result := tickList.ToArray;
-                finally
-                    tickList.Free;
-                end;
-            end
-        );
 end;
 
 function TAuraTABFileServer.ParseFileName(const FileName: string): TAuraDataFile;
@@ -546,7 +509,7 @@ begin
     if symbol = '' then
         exit;
 
-    Result := TAuraDataFile.Create(path, symbol, fileExt, year, month);
+    Result := TAuraDataFile.Create(path, symbol, fileExt, year, month, baseName);
 end;
 
 class function TAuraTABFileServer.ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>;
@@ -628,7 +591,193 @@ begin
             if bytesRead <> SizeOf(TFileRecord) then
                 raise EReadError.CreateFmt('Read error. Expected %d bytes, read %d.', [SizeOf(TFileRecord), bytesRead]);
 
-            Result[chunkIndex][indexInChunk].Create(rec.TimeStamp, rec.Data);
+            // Corrected syntax: Assign the newly created record instance.
+            Result[chunkIndex][indexInChunk] := TDataPoint<TAuraAskBidFileItem>.Create(rec.TimeStamp, rec.Data);
+            Inc(indexInChunk);
+        end;
+    end;
+end;
+
+{ TAuraM1FileServer }
+
+function TAuraM1FileServer.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>;
+begin
+    Result := TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>.Null;
+
+    if not TFile.Exists(FileName) then
+        exit;
+
+    var capFileName := FileName;
+    // M1 files are always compressed with the .m1 extension
+    if FileName.EndsWith('.m1', True) then
+    begin
+        Result :=
+            TFuture<TBytes>
+                .Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
+                .Chain<TArray<TArray<TDataPoint<TOhlcItem>>>>(
+                    function(bytes: TBytes): TArray<TArray<TDataPoint<TOhlcItem>>>
+                    begin
+                        var compMemoryStream := TBytesStream.Create(bytes);
+                        try
+                            compMemoryStream.Position := 0;
+                            Result := ReadCompressedData(compMemoryStream);
+                        finally
+                            compMemoryStream.Free;
+                        end;
+                    end);
+    end;
+end;
+
+function TAuraM1FileServer.ParseFileName(const FileName: string): TAuraDataFile;
+var
+    path, fileNameNoPath, baseName, ext, symbol: string;
+    year, month: Integer;
+    parts, dateParts: TArray<string>;
+begin
+    Result := Default(TAuraDataFile);
+    path := TPath.GetDirectoryName(FileName);
+    fileNameNoPath := TPath.GetFileName(FileName);
+
+    // M1 files are compressed and have a .m1 extension
+    if not fileNameNoPath.EndsWith('.m1', True) then
+        exit;
+
+    ext := TPath.GetExtension(fileNameNoPath);
+    baseName := TPath.GetFileNameWithoutExtension(fileNameNoPath);
+
+    // Expected format: SYMBOL_YYYY-MM_YYYY-MM
+    parts := baseName.Split(['_']);
+    if Length(parts) < 3 then
+        exit; // Not enough parts for SYMBOL_START_END
+
+    // The symbol can contain underscores, so we take everything before the last two parts.
+    var startDatePart := parts[High(parts) - 1];
+
+    dateParts := startDatePart.Split(['-']);
+    if Length(dateParts) <> 2 then
+        exit; // Start date is not YYYY-MM
+
+    if not TryStrToInt(dateParts[0], year) then
+        exit;
+
+    if not TryStrToInt(dateParts[1], month) then
+        exit;
+
+    if (month < 1) or (month > 12) or (year <= 0) then
+        exit;
+
+    symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
+    if symbol = '' then
+        exit;
+
+    // The TAuraDataFile only represents the start date of the file batch.
+    Result := TAuraDataFile.Create(path, symbol, ext, year, month, baseName);
+end;
+
+class function TAuraM1FileServer.ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
+var
+    decompressionStream: TStream;
+    localHeader: TZipHeader;
+    entryIndex, i: Integer;
+    zipFileInstance: TZipFile;
+begin
+    Result := nil;
+    decompressionStream := nil;
+    zipFileInstance := nil;
+
+    // M1 files are now zip archives containing the actual data file.
+    if not Assigned(InputStream) or (InputStream.Size = 0) then
+        exit;
+
+    try
+        InputStream.Position := 0;
+        zipFileInstance := TZipFile.Create;
+        zipFileInstance.Open(InputStream, TZipMode.zmRead);
+
+        if zipFileInstance.FileCount = 0 then
+            exit;
+
+        // Find the data file entry within the archive.
+        // The C# code creates an entry with the same name as the base filename, ending in .m1.
+        entryIndex := -1;
+        for i := 0 to zipFileInstance.FileCount - 1 do
+        begin
+            if zipFileInstance.FileNames[i].EndsWith('.m1', True) then
+            begin
+                entryIndex := i;
+                break;
+            end;
+        end;
+
+        // Fallback to the first entry if no specific .m1 file is found inside.
+        if entryIndex = -1 then
+            entryIndex := 0;
+
+        zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
+        if not Assigned(decompressionStream) then
+            exit;
+
+        // The decompressed stream now contains the raw binary data.
+        Result := ReadUncompressedData(decompressionStream);
+    finally
+        decompressionStream.Free;
+        zipFileInstance.Free;
+    end;
+end;
+
+class function TAuraM1FileServer.ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
+type
+    TFileRecord = TAuraM1FileItem;
+var
+    fileSize: Int64;
+    totalRecordCount, bytesRead, chunkIndex, indexInChunk, recordIndex: Integer;
+    rec: TFileRecord;
+    ohlc: TOhlcItem;
+    factor: Double;
+begin
+    Result := nil;
+    InputStream.Position := 0;
+    fileSize := InputStream.Size;
+    if (fileSize = 0) or ((fileSize mod SizeOf(TFileRecord)) <> 0) then
+        exit;
+
+    totalRecordCount := fileSize div SizeOf(TFileRecord);
+    if totalRecordCount > 0 then
+    begin
+        var numChunks := (totalRecordCount + ChunkSize - 1) div ChunkSize;
+        SetLength(Result, numChunks);
+        chunkIndex := 0;
+        indexInChunk := 0;
+        SetLength(Result[chunkIndex], Min(ChunkSize, totalRecordCount));
+
+        for recordIndex := 0 to totalRecordCount - 1 do
+        begin
+            if indexInChunk >= ChunkSize then
+            begin
+                Inc(chunkIndex);
+                indexInChunk := 0;
+                var remainingRecords := totalRecordCount - (chunkIndex * ChunkSize);
+                SetLength(Result[chunkIndex], Min(ChunkSize, remainingRecords));
+            end;
+
+            bytesRead := InputStream.Read(rec, SizeOf(TFileRecord));
+            if bytesRead <> SizeOf(TFileRecord) then
+                raise EReadError.CreateFmt('Read error. Expected %d bytes, read %d.', [SizeOf(TFileRecord), bytesRead]);
+
+            // Convert from M1 file record to a standard OHLC item
+            if rec.Digits > 0 then
+                factor := Power(10, rec.Digits)
+            else
+                factor := 1.0;
+
+            ohlc.Open := rec.Open / factor;
+            ohlc.High := rec.High / factor;
+            ohlc.Low := rec.Low / factor;
+            ohlc.Close := rec.Close / factor;
+            ohlc.Volume := rec.TickVolume;
+
+            // Corrected syntax: Assign the newly created record instance.
+            Result[chunkIndex][indexInChunk] := TDataPoint<TOhlcItem>.Create(rec.OADateTime, ohlc);
             Inc(indexInChunk);
         end;
     end;