integrated TFuture

ChartDisplay/ChartDisplayMain.dfm

@@ -41,9 +41,7 @@ object ChartForm: TChartForm
     Height = 23
     Anchors = [akLeft, akTop, akRight]
     TabOrder = 0
-    Text = 
+    Text = '\\COFFEE\TickData\Pepperstone\EURUSD_2017_02.tab_zip'
-      'C:\Users\Brummel\Documents\cAlgo\TickData\Pepperstone\EURUSD_201' +
-      '7.tab'
   end
   object LoadButton: TButton
     Left = 686

ChartDisplay/ChartDisplayMain.pas

@@ -543,12 +543,13 @@ var
     InitialDirectory: string;
     SelectedPath, SelectedSymbol: string;
     SelectedYear: Integer;
+    SelectedMonth: Integer;
     IsValidSyntax: Boolean;
 begin
     OpenDialog := TOpenDialog.Create( Self );
     try
         OpenDialog.Title := 'Open Tick Data File';
-        OpenDialog.Filter := 'Tick Data Files (*.tab)|*.TAB|All Files (*.*)|*.*';
+        OpenDialog.Filter := 'Tick Data Files (*.tab)|*.tab;*.tab_zip;*.tab_live|All Files (*.*)|*.*';
         OpenDialog.DefaultExt := 'tab';
         OpenDialog.Options := [ofFileMustExist, ofPathMustExist, ofHideReadOnly, ofEnableSizing];
 
@@ -574,11 +575,11 @@ begin
             Memo.Lines.Add( FormatDateTime( 'yyyy-mm-dd hh:nn:ss', Now ) + ' - File selected via dialog: ' + OpenDialog.FileName );
 
             IsValidSyntax := Myc.TickLoader.TryParseFileName( OpenDialog.FileName,
-              SelectedPath, SelectedSymbol, SelectedYear );
+              SelectedPath, SelectedSymbol, SelectedYear, SelectedMonth );
 
             if not IsValidSyntax then
             begin
-                MessageDlg( Format( 'Warning: The selected filename ''%s'' does not match the expected format (Prefix_Symbol_Year.tab).',
+                MessageDlg( Format( 'Warning: The selected filename ''%s'' does not match the expected format (Prefix_Symbol_Year_Month.tab).',
                   [System.IOUtils.TPath.GetFileName( OpenDialog.FileName )] ), mtWarning, [mbOK], 0 );
                 Memo.Lines.Add( 'Warning: Incorrect filename syntax chosen by user.' );
             end;

ChartDisplay/ChartDisplayProject.dpr

@@ -1,10 +1,12 @@
 program ChartDisplayProject;
 
 uses
+  FastMM5,
   Vcl.Forms,
   ChartDisplayMain in 'ChartDisplayMain.pas' {ChartForm},
   Myc.OHLCCache in '..\common\Myc.OHLCCache.pas',
-  Myc.ChartDataController in '..\common\Myc.ChartDataController.pas';
+  Myc.ChartDataController in '..\common\Myc.ChartDataController.pas',
+  DataStreamer in 'DataStreamer.pas';
 
 {$R *.res}
 

ChartDisplay/ChartDisplayProject.dproj

@@ -71,7 +71,7 @@
         <UWP_DelphiLogo44>$(BDS)\bin\Artwork\Windows\UWP\delphi_UwpDefault_44.png</UWP_DelphiLogo44>
         <UWP_DelphiLogo150>$(BDS)\bin\Artwork\Windows\UWP\delphi_UwpDefault_150.png</UWP_DelphiLogo150>
         <SanitizedProjectName>ChartDisplayProject</SanitizedProjectName>
-        <DCC_UnitSearchPath>..\common;$(DCC_UnitSearchPath)</DCC_UnitSearchPath>
+        <DCC_UnitSearchPath>..\common;T:\Myc\Src;$(DCC_UnitSearchPath)</DCC_UnitSearchPath>
         <VerInfo_Locale>1031</VerInfo_Locale>
         <VerInfo_Keys>CompanyName=;FileDescription=$(MSBuildProjectName);FileVersion=1.0.0.0;InternalName=;LegalCopyright=;LegalTrademarks=;OriginalFilename=;ProgramID=com.embarcadero.$(MSBuildProjectName);ProductName=$(MSBuildProjectName);ProductVersion=1.0.0.0;Comments=</VerInfo_Keys>
     </PropertyGroup>
@@ -110,6 +110,9 @@
     </PropertyGroup>
     <PropertyGroup Condition="'$(Cfg_1_Win64)'!=''">
         <AppDPIAwarenessMode>PerMonitorV2</AppDPIAwarenessMode>
+        <DCC_RunTimeTypeInfo>true</DCC_RunTimeTypeInfo>
+        <VerInfo_IncludeVerInfo>true</VerInfo_IncludeVerInfo>
+        <VerInfo_Locale>1033</VerInfo_Locale>
     </PropertyGroup>
     <PropertyGroup Condition="'$(Cfg_2)'!=''">
         <DCC_LocalDebugSymbols>false</DCC_LocalDebugSymbols>
@@ -129,10 +132,10 @@
         </DelphiCompile>
         <DCCReference Include="ChartDisplayMain.pas">
             <Form>ChartForm</Form>
-            <FormType>dfm</FormType>
         </DCCReference>
         <DCCReference Include="..\common\Myc.OHLCCache.pas"/>
         <DCCReference Include="..\common\Myc.ChartDataController.pas"/>
+        <DCCReference Include="DataStreamer.pas"/>
         <BuildConfiguration Include="Base">
             <Key>Base</Key>
         </BuildConfiguration>
@@ -153,12 +156,7 @@
                 <Source>
                     <Source Name="MainSource">ChartDisplayProject.dpr</Source>
                 </Source>
-                <Excluded_Packages>
+                <Excluded_Packages/>
-                    <Excluded_Packages Name="$(BDSBIN)\bcboffice2k290.bpl">Embarcadero C++Builder Office 2000 Servers Package</Excluded_Packages>
-                    <Excluded_Packages Name="$(BDSBIN)\bcbofficexp290.bpl">Embarcadero C++Builder Office XP Servers Package</Excluded_Packages>
-                    <Excluded_Packages Name="$(BDSBIN)\dcloffice2k290.bpl">Microsoft Office 2000 Sample Automation Server Wrapper Components</Excluded_Packages>
-                    <Excluded_Packages Name="$(BDSBIN)\dclofficexp290.bpl">Microsoft Office XP Sample Automation Server Wrapper Components</Excluded_Packages>
-                </Excluded_Packages>
             </Delphi.Personality>
             <Deployment Version="5">
                 <DeployFile LocalName="Win32\Debug\ChartDisplayProject.exe" Configuration="Debug" Class="ProjectOutput">
@@ -173,9 +171,9 @@
                         <Overwrite>true</Overwrite>
                     </Platform>
                 </DeployFile>
-                <DeployFile LocalName="Win64\Debug\ChartDisplayProject.rsm" Configuration="Debug" Class="DebugSymbols">
+                <DeployFile LocalName="Win64\Release\ChartDisplayProject.exe" Configuration="Release" Class="ProjectOutput">
                     <Platform Name="Win64">
-                        <RemoteName>ChartDisplayProject.rsm</RemoteName>
+                        <RemoteName>ChartDisplayProject.exe</RemoteName>
                         <Overwrite>true</Overwrite>
                     </Platform>
                 </DeployFile>

ChartDisplay/DataStreamer.pas

@@ -0,0 +1,671 @@
+unit DataStreamer;
+
+interface
+
+uses
+    System.SysUtils, System.Classes, System.Generics.Collections, System.IOUtils, System.Threading,
+    System.Zip;
+
+type
+    IDataPage = interface
+        ['{B3E3C5B7-7C1A-4A8B-8B7E-15C9D9E0F4A6}']
+        function GetStream: TStream;
+        function GetSourceIdentifier: string;
+        function GetLastModificationTime: TDateTime;
+    end;
+
+    IRecordStreamReader<T: record> = interface
+        ['{C1B7E8F0-5A0A-4F3B-BD28-18D5E5A3F7A0}']
+        function GetCurrentRecord: T;
+        function MoveToNextRecord: Boolean;
+        procedure Reset;
+        function GetCurrentRecordIndex: Int64;
+        function Seek(RecordIndex: Int64): Int64;
+
+        property CurrentRecord: T read GetCurrentRecord;
+        property CurrentRecordIndex: Int64 read GetCurrentRecordIndex;
+    end;
+
+    TLocalFilePage = class(TInterfacedObject, IDataPage)
+    private
+        FActualFilePath: string;
+        FMemoryStream: TMemoryStream;
+        FLastModifiedOnDisk: TDateTime;
+
+        function GetStream: TStream;
+        function GetSourceIdentifier: string;
+        function GetLastModificationTime: TDateTime;
+    public
+        constructor Create(const AActualFilePath: string; ADataStream: TMemoryStream; ALastModifiedTimeOnDisk: TDateTime);
+        destructor Destroy; override;
+        class function IsFileModified(const AFileName: string; KnownLastModified: TDateTime): Boolean;
+    end;
+
+    TStreamingFileRecordReader<T: record> = class(TInterfacedObject, IRecordStreamReader<T>)
+    private
+        FDirectoryPath: string;
+        FDataSetName: string;
+        FExtension: string;
+
+        FFilePaths: TStringList;
+        FCachedPages: TObjectDictionary<string, IDataPage>;
+        FPageCacheQueue: TQueue<string>;
+        FMaxCacheSize: Integer;
+        FPreloadPageTask: ITask;
+        FLockObject: TObject;
+
+        FCurrentPage: IDataPage;
+        FCurrentPageStream: TStream;
+        FCurrentPageIndexInFileList: Integer;
+
+        FRecordSize: Integer;
+        FCurrentRecord: T;
+        FRecordsInCurrentPage: Int64;
+        FRecordsReadFromCurrentPage: Int64;
+        FGlobalRecordIndex: Int64;
+
+        procedure ScanDirectoryAndMatchFiles;
+        function LoadPage(ActualFilePath: string): IDataPage;
+        procedure EnsurePageInCache(ActualFilePath: string; IsPreload: Boolean = False);
+        procedure EvictOldestPage;
+        procedure DoPreloadWork(const APathToPreload: string);
+        procedure StartPreloadNextPage;
+        procedure InvalidatePageIfModified(PageToCheck: IDataPage; IsCurrentPage: Boolean);
+        function ActivatePage(PageIndexInFileList: Integer): Boolean;
+        function GetRecordCountForFile(ActualFilePath: string): Int64;
+
+        function GetCurrentRecord: T;
+        function MoveToNextRecord: Boolean;
+        procedure Reset;
+        function GetCurrentRecordIndex: Int64;
+        function Seek(RecordIndex: Int64): Int64;
+    public
+        constructor Create(const ADirectoryPath: string; const ADataSetName: string; const AExtension: string; AMaxCacheSize: Integer = 2);
+        destructor Destroy; override;
+    end;
+
+implementation
+
+uses
+    System.DateUtils, System.Types, System.StrUtils,
+    System.Math;
+
+{ TLocalFilePage }
+
+constructor TLocalFilePage.Create(const AActualFilePath: string; ADataStream: TMemoryStream; ALastModifiedTimeOnDisk: TDateTime);
+begin
+    inherited Create;
+    FActualFilePath := AActualFilePath;
+    FMemoryStream := ADataStream;
+    FMemoryStream.Position := 0;
+    FLastModifiedOnDisk := ALastModifiedTimeOnDisk;
+end;
+
+destructor TLocalFilePage.Destroy;
+begin
+    FMemoryStream.Free;
+    inherited Destroy;
+end;
+
+function TLocalFilePage.GetStream: TStream;
+begin
+    FMemoryStream.Position := 0;
+    result := FMemoryStream;
+end;
+
+function TLocalFilePage.GetSourceIdentifier: string;
+begin
+    result := FActualFilePath;
+end;
+
+function TLocalFilePage.GetLastModificationTime: TDateTime;
+begin
+    result := FLastModifiedOnDisk;
+end;
+
+class function TLocalFilePage.IsFileModified(const AFileName: string; KnownLastModified: TDateTime): Boolean;
+var
+    currentModTime: TDateTime;
+begin
+    if TFile.Exists(AFileName) then
+    begin
+        currentModTime := TFile.GetLastWriteTime(AFileName);
+        result := Abs(MilliSecondsBetween(currentModTime, KnownLastModified)) > 1000;
+    end
+    else
+    begin
+        result := True;
+    end;
+end;
+
+{ TStreamingFileRecordReader<T> }
+
+constructor TStreamingFileRecordReader<T>.Create(const ADirectoryPath: string; const ADataSetName: string; const AExtension: string; AMaxCacheSize: Integer);
+begin
+    inherited Create;
+    FLockObject := TObject.Create;
+
+    if not TDirectory.Exists(ADirectoryPath) then
+        raise EDirectoryNotFoundException.Create('Directory not found: ' + ADirectoryPath);
+    if Trim(ADataSetName) = '' then
+        raise EArgumentException.Create('DataSetName cannot be empty.');
+    if Trim(AExtension) = '' then
+        raise EArgumentException.Create('Extension cannot be empty.');
+
+    FDirectoryPath := IncludeTrailingPathDelimiter(ADirectoryPath);
+    FDataSetName := ADataSetName;
+    FExtension := AExtension;
+    if StartsText('.', FExtension) then
+       FExtension := Copy(FExtension, 2, Length(FExtension) -1);
+
+    FFilePaths := TStringList.Create;
+    FFilePaths.Sorted := True;
+
+    FCachedPages := TObjectDictionary<string, IDataPage>.Create([doOwnsValues]);
+    FPageCacheQueue := TQueue<string>.Create;
+    FMaxCacheSize := System.Math.Max(2, AMaxCacheSize);
+    FRecordSize := SizeOf(T);
+    if FRecordSize = 0 then
+        raise EArgumentException.Create('Record size for type T cannot be zero.');
+
+    FCurrentPageIndexInFileList := -1;
+    FGlobalRecordIndex := -1;
+    FRecordsReadFromCurrentPage := 0;
+    FRecordsInCurrentPage := 0;
+    FCurrentPage := nil;
+    FCurrentPageStream := nil;
+
+    ScanDirectoryAndMatchFiles;
+    Reset;
+end;
+
+destructor TStreamingFileRecordReader<T>.Destroy;
+var
+    taskToWaitFor: ITask;
+begin
+    taskToWaitFor := nil;
+    TMonitor.Enter(FLockObject);
+    try
+        taskToWaitFor := FPreloadPageTask;
+        FPreloadPageTask := nil;
+    finally
+        TMonitor.Exit(FLockObject);
+    end;
+
+    if (taskToWaitFor <> nil) and (taskToWaitFor.Status in [TTaskStatus.Created, TTaskStatus.WaitingToRun, TTaskStatus.Running]) then
+    begin
+        // Consider taskToWaitFor.Cancel; taskToWaitFor.Wait;
+    end;
+
+    FFilePaths.Free;
+    FCachedPages.Free;
+    FPageCacheQueue.Free;
+    FLockObject.Free;
+    inherited Destroy;
+end;
+
+procedure TStreamingFileRecordReader<T>.ScanDirectoryAndMatchFiles;
+var
+    allFiles: TStringDynArray;
+    fileNameOnly, filePath: string;
+    expectedPrefix, baseExtensionPattern, zipExtensionPattern: string;
+begin
+    TMonitor.Enter(FLockObject);
+    try
+        FFilePaths.Clear;
+        allFiles := TDirectory.GetFiles(FDirectoryPath);
+        expectedPrefix := FDataSetName + '_';
+        baseExtensionPattern := '.' + FExtension;
+        zipExtensionPattern := baseExtensionPattern + '_zip';
+
+        for filePath in allFiles do
+        begin
+            fileNameOnly := ExtractFileName(filePath);
+            if StartsText(expectedPrefix, fileNameOnly) then
+            begin
+                if EndsText(zipExtensionPattern, fileNameOnly.ToLower) or EndsText(baseExtensionPattern, fileNameOnly.ToLower) then
+                begin
+                    FFilePaths.Add(filePath);
+                end;
+            end;
+        end;
+    finally
+        TMonitor.Exit(FLockObject);
+    end;
+end;
+
+function TStreamingFileRecordReader<T>.LoadPage(ActualFilePath: string): IDataPage;
+var
+    dataStream: TMemoryStream;
+    fileLastModified: TDateTime;
+    zipFile: TZipFile;
+    isCompressed: Boolean;
+    fileExtensionOnDisk: string;
+    decompressionStream: TStream;
+    localHeader: TZipHeader;
+    fileNameInZip: string;
+begin
+    dataStream := TMemoryStream.Create;
+    try
+        fileLastModified := TFile.GetLastWriteTime(ActualFilePath);
+        fileExtensionOnDisk := ExtractFileExt(ActualFilePath);
+        isCompressed := EndsText('_zip', fileExtensionOnDisk.ToLower);
+
+        if isCompressed then
+        begin
+            zipFile := TZipFile.Create;
+            try
+                zipFile.Open(ActualFilePath, zmRead);
+                if zipFile.FileCount > 0 then
+                begin
+                    fileNameInZip := zipFile.FileNames[0];
+                    zipFile.Read(fileNameInZip, decompressionStream, localHeader);
+                    try
+                        dataStream.CopyFrom(decompressionStream, 0);
+                    finally
+                        decompressionStream.Free;
+                    end;
+                end
+                else
+                    raise Exception.Create('Compressed file is empty: ' + ActualFilePath);
+            finally
+                zipFile.Free;
+            end;
+        end
+        else
+            dataStream.LoadFromFile(ActualFilePath);
+
+        dataStream.Position := 0;
+        result := TLocalFilePage.Create(ActualFilePath, dataStream, fileLastModified);
+    except
+        dataStream.Free;
+        raise;
+    end;
+end;
+
+procedure TStreamingFileRecordReader<T>.EnsurePageInCache(ActualFilePath: string; IsPreload: Boolean);
+var
+    page: IDataPage;
+    newlyLoadedPage: IDataPage;
+begin
+    // Assumes FLockObject is held by the caller
+    if not FCachedPages.TryGetValue(ActualFilePath, page) then
+    begin
+        if FCachedPages.Count >= FMaxCacheSize then
+            EvictOldestPage;
+        newlyLoadedPage := LoadPage(ActualFilePath);
+        FCachedPages.Add(ActualFilePath, newlyLoadedPage);
+        FPageCacheQueue.Enqueue(ActualFilePath);
+    end
+    else
+    begin
+        if not IsPreload then
+            InvalidatePageIfModified(page, page = FCurrentPage);
+    end;
+end;
+
+procedure TStreamingFileRecordReader<T>.EvictOldestPage;
+var
+    oldestFilePath: string;
+begin
+    // Assumes FLockObject is held by the caller
+    if FPageCacheQueue.Count > 0 then
+    begin
+        oldestFilePath := FPageCacheQueue.Dequeue;
+        if FCachedPages.ContainsKey(oldestFilePath) then
+            FCachedPages.Remove(oldestFilePath);
+    end;
+end;
+
+procedure TStreamingFileRecordReader<T>.InvalidatePageIfModified(PageToCheck: IDataPage; IsCurrentPage: Boolean);
+var
+    actualFilePath: string;
+    reloadedPage: IDataPage;
+begin
+    // Assumes FLockObject is held by the caller
+    if PageToCheck = nil then exit;
+
+    actualFilePath := PageToCheck.GetSourceIdentifier;
+
+    if TLocalFilePage.IsFileModified(actualFilePath, PageToCheck.GetLastModificationTime) then
+    begin
+        FCachedPages.Remove(actualFilePath);
+
+        if IsCurrentPage then
+        begin
+            FCurrentPage := nil;
+            FCurrentPageStream := nil;
+
+            reloadedPage := LoadPage(actualFilePath);
+            FCachedPages.Add(actualFilePath, reloadedPage);
+
+            FCurrentPage := reloadedPage;
+            FCurrentPageStream := FCurrentPage.GetStream;
+            if (FCurrentPageStream <> nil) and (FCurrentPageStream.Size >= FRecordSize) then
+                FRecordsInCurrentPage := FCurrentPageStream.Size div FRecordSize
+            else
+                FRecordsInCurrentPage := 0;
+
+            if (FRecordsReadFromCurrentPage * FRecordSize < FCurrentPageStream.Size) and (FRecordsReadFromCurrentPage <= FRecordsInCurrentPage) then
+                FCurrentPageStream.Position := FRecordsReadFromCurrentPage * FRecordSize
+            else
+            begin
+                FCurrentPageStream.Position := 0;
+                FRecordsReadFromCurrentPage := 0;
+            end;
+        end;
+    end;
+end;
+
+procedure TStreamingFileRecordReader<T>.DoPreloadWork(const APathToPreload: string);
+begin
+    try
+        TMonitor.Enter(FLockObject);
+        try
+            if not FCachedPages.ContainsKey(APathToPreload) then
+                EnsurePageInCache(APathToPreload, True);
+        finally
+            TMonitor.Exit(FLockObject);
+        end;
+    except
+        on E: Exception do
+        begin
+            // Optional: Log error E.Message
+        end;
+    end;
+end;
+
+procedure TStreamingFileRecordReader<T>.StartPreloadNextPage;
+var
+    nextPageIndexToLoad: Integer;
+    nextFilePath: string;
+    currentTask: ITask;
+begin
+    TMonitor.Enter(FLockObject);
+    try
+        currentTask := FPreloadPageTask;
+        if (currentTask <> nil) and (currentTask.Status in [TTaskStatus.Created, TTaskStatus.WaitingToRun, TTaskStatus.Running]) then
+            exit;
+
+        nextPageIndexToLoad := FCurrentPageIndexInFileList + 1;
+        if nextPageIndexToLoad < FFilePaths.Count then
+        begin
+            nextFilePath := FFilePaths[nextPageIndexToLoad];
+            if FCachedPages.ContainsKey(nextFilePath) then
+                exit;
+
+            FPreloadPageTask := TTask.Run(
+                procedure
+                begin
+                    DoPreloadWork(nextFilePath);
+                end
+            );
+        end;
+    finally
+        TMonitor.Exit(FLockObject);
+    end;
+end;
+
+function TStreamingFileRecordReader<T>.ActivatePage(PageIndexInFileList: Integer): Boolean;
+var
+    actualFilePath: string;
+    pageFromCache: IDataPage;
+begin
+    result := False;
+    if (PageIndexInFileList < 0) or (PageIndexInFileList >= FFilePaths.Count) then
+    begin FCurrentPage := nil; FCurrentPageStream := nil; exit; end;
+
+    actualFilePath := FFilePaths[PageIndexInFileList];
+
+    TMonitor.Enter(FLockObject);
+    try
+        EnsurePageInCache(actualFilePath, False);
+        if not FCachedPages.TryGetValue(actualFilePath, pageFromCache) then
+        begin FCurrentPage := nil; FCurrentPageStream := nil; exit; end;
+
+        FCurrentPage := pageFromCache;
+        FCurrentPageStream := FCurrentPage.GetStream;
+        FCurrentPageIndexInFileList := PageIndexInFileList;
+        FRecordsReadFromCurrentPage := 0;
+        if (FCurrentPageStream <> nil) and (FCurrentPageStream.Size >= FRecordSize) then
+            FRecordsInCurrentPage := FCurrentPageStream.Size div FRecordSize
+        else
+            FRecordsInCurrentPage := 0;
+
+        if FRecordsInCurrentPage > 0 then
+        begin
+            result := True;
+            StartPreloadNextPage;
+        end;
+    finally
+        TMonitor.Exit(FLockObject);
+    end;
+end;
+
+function TStreamingFileRecordReader<T>.GetRecordCountForFile(ActualFilePath: string): Int64;
+var
+  page: IDataPage;
+  stream: TStream;
+begin
+  result := 0;
+  EnsurePageInCache(ActualFilePath, False); // Assumes FLockObject is held by caller
+  if FCachedPages.TryGetValue(ActualFilePath, page) then
+  begin
+    stream := page.GetStream;
+    if (stream <> nil) and (stream.Size >= FRecordSize) then
+      result := stream.Size div FRecordSize;
+  end;
+end;
+
+function TStreamingFileRecordReader<T>.GetCurrentRecord: T;
+begin
+    if FGlobalRecordIndex < 0 then
+      Result := System.Default(T)
+    else
+      Result := FCurrentRecord;
+end;
+
+function TStreamingFileRecordReader<T>.GetCurrentRecordIndex: Int64;
+begin
+    result := FGlobalRecordIndex;
+end;
+
+function TStreamingFileRecordReader<T>.MoveToNextRecord: Boolean;
+var
+    bytesRead: NativeInt; // ReadData<T> returns NativeInt
+    attemptedNextPageActivation: Boolean;
+begin
+    result := False;
+    attemptedNextPageActivation := False;
+
+    TMonitor.Enter(FLockObject);
+    try
+        while True do
+        begin
+            if FCurrentPageStream = nil then
+            begin
+                if attemptedNextPageActivation or (FCurrentPageIndexInFileList >= FFilePaths.Count - 1) then
+                begin FGlobalRecordIndex := -1; exit; end;
+                if not ActivatePage(FCurrentPageIndexInFileList + 1) then
+                begin FGlobalRecordIndex := -1; exit; end;
+                attemptedNextPageActivation := True;
+            end;
+
+            if FRecordsReadFromCurrentPage >= FRecordsInCurrentPage then
+            begin
+                if attemptedNextPageActivation or (FCurrentPageIndexInFileList >= FFilePaths.Count - 1) then
+                begin FGlobalRecordIndex := -1; exit; end;
+                FCurrentPageStream := nil;
+                if not ActivatePage(FCurrentPageIndexInFileList + 1) then
+                begin FGlobalRecordIndex := -1; exit; end;
+                attemptedNextPageActivation := True;
+                continue;
+            end;
+
+            FCurrentPageStream.Position := FRecordsReadFromCurrentPage * FRecordSize;
+            if (FCurrentPageStream.Size - FCurrentPageStream.Position) < FRecordSize then
+            begin
+                FRecordsReadFromCurrentPage := FRecordsInCurrentPage;
+                attemptedNextPageActivation := False;
+                FCurrentPageStream := nil;
+                continue;
+            end;
+
+            // Verwende die generische TStream.ReadData<T> Methode
+            bytesRead := FCurrentPageStream.ReadData<T>(FCurrentRecord); // User line 518 context
+
+            if bytesRead = FRecordSize then // FRecordSize is SizeOf(T)
+            begin
+                Inc(FRecordsReadFromCurrentPage);
+                if FGlobalRecordIndex < 0 then
+                    FGlobalRecordIndex := 0
+                else
+                    Inc(FGlobalRecordIndex); // << User line 518 (E2010), hopefully fixed by ReadData<T>
+                result := True; exit;
+            end
+            else
+            begin
+                FRecordsReadFromCurrentPage := FRecordsInCurrentPage;
+                attemptedNextPageActivation := False;
+                FCurrentPageStream := nil;
+                continue;
+            end;
+        end;
+    finally
+        TMonitor.Exit(FLockObject);
+    end;
+end;
+
+procedure TStreamingFileRecordReader<T>.Reset;
+var
+    taskToCheck: ITask;
+begin
+    TMonitor.Enter(FLockObject);
+    try
+        taskToCheck := FPreloadPageTask;
+        if (taskToCheck <> nil) and (taskToCheck.Status in [TTaskStatus.Created, TTaskStatus.WaitingToRun, TTaskStatus.Running]) then
+        begin
+            // Cancellation logic
+        end;
+        FPreloadPageTask := nil;
+
+        FCurrentPageIndexInFileList := -1;
+        FGlobalRecordIndex := -1;
+        FRecordsReadFromCurrentPage := 0;
+        FRecordsInCurrentPage := 0;
+        FCurrentPage := nil;
+        FCurrentPageStream := nil;
+        FCurrentRecord := System.Default(T);
+        if FFilePaths.Count > 0 then
+            if not ActivatePage(0) then;
+    finally
+        TMonitor.Exit(FLockObject);
+    end;
+end;
+
+function TStreamingFileRecordReader<T>.Seek(RecordIndex: Int64): Int64;
+var
+    pageIdx: Integer;
+    cumulativeRecordOffset: Int64;
+    recordsInCurrentFile: Int64;
+    recordOffsetInPage: Int64;
+    effectiveTargetIdx: Int64;
+    i: Integer;
+    sumRecordsBeforeThisPage: Int64;
+    bytesRead: NativeInt; // For ReadData<T>
+begin
+    TMonitor.Enter(FLockObject);
+    try
+        if FFilePaths.Count = 0 then
+        begin
+            FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
+            FCurrentRecord := System.Default(T);
+            result := -1; exit;
+        end;
+
+        effectiveTargetIdx := RecordIndex;
+        if effectiveTargetIdx < 0 then effectiveTargetIdx := 0;
+
+        cumulativeRecordOffset := 0;
+        for pageIdx := 0 to FFilePaths.Count - 1 do
+        begin
+            recordsInCurrentFile := GetRecordCountForFile(FFilePaths[pageIdx]);
+            if effectiveTargetIdx < cumulativeRecordOffset + recordsInCurrentFile then
+            begin
+                if not ActivatePage(pageIdx) then
+                begin
+                    FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
+                    FCurrentRecord := System.Default(T);
+                    result := -1; exit;
+                end;
+                recordOffsetInPage := effectiveTargetIdx - cumulativeRecordOffset;
+                FCurrentPageStream.Position := recordOffsetInPage * FRecordSize;
+
+                // Verwende die generische TStream.ReadData<T> Methode
+                bytesRead := FCurrentPageStream.ReadData<T>(FCurrentRecord);
+
+                if bytesRead = FRecordSize then
+                begin
+                    FGlobalRecordIndex := effectiveTargetIdx; FRecordsReadFromCurrentPage := recordOffsetInPage + 1;
+                    result := FGlobalRecordIndex; exit;
+                end
+                else
+                begin // User line ~604 context (E2008)
+                    FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
+                    FCurrentRecord := System.Default(T);
+                    result := -1; exit;
+                end;
+            end;
+            inc(cumulativeRecordOffset, recordsInCurrentFile);
+        end;
+
+        if cumulativeRecordOffset = 0 then // User line ~616 context (E2029) for assignment after this block
+        begin
+            FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
+            FCurrentRecord := System.Default(T);
+            result := -1; exit;
+        end;
+
+        for pageIdx := FFilePaths.Count - 1 downto 0 do
+        begin
+            recordsInCurrentFile := GetRecordCountForFile(FFilePaths[pageIdx]);
+            if recordsInCurrentFile > 0 then
+            begin
+                if not ActivatePage(pageIdx) then
+                begin
+                    FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
+                    FCurrentRecord := System.Default(T);
+                    result := -1; exit;
+                end;
+                recordOffsetInPage := recordsInCurrentFile - 1;
+                FCurrentPageStream.Position := recordOffsetInPage * FRecordSize;
+
+                // Verwende die generische TStream.ReadData<T> Methode
+                bytesRead := FCurrentPageStream.ReadData<T>(FCurrentRecord);
+
+                if bytesRead = FRecordSize then
+                begin
+                    sumRecordsBeforeThisPage := 0;
+                    for i := 0 to pageIdx - 1 do
+                        sumRecordsBeforeThisPage := sumRecordsBeforeThisPage + GetRecordCountForFile(FFilePaths[i]);
+                    FGlobalRecordIndex := sumRecordsBeforeThisPage + recordOffsetInPage; // User line 616 context (E2029/E2014)
+                    FRecordsReadFromCurrentPage := recordOffsetInPage + 1;
+                    result := FGlobalRecordIndex; exit;
+                end
+                else
+                begin // User line ~639 context (E2008)
+                    FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
+                    FCurrentRecord := System.Default(T);
+                    result := -1; exit;
+                end;
+            end;
+        end;
+        FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
+        FCurrentRecord := System.Default(T);
+        result := -1; exit;
+    finally
+        TMonitor.Exit(FLockObject);
+    end;
+end;
+
+end.

DartaLoader.mm

@@ -0,0 +1,127 @@
+<map version="freeplane 1.12.1">
+<!--To view this file, download free mind mapping software Freeplane from https://www.freeplane.org -->
+<node TEXT="Data loader&#xa;" FOLDED="false" ID="1" CREATED="1748275449538" MODIFIED="1748275581071"><hook NAME="MapStyle">
+    <properties edgeColorConfiguration="#808080ff,#ff0000ff,#0000ffff,#00ff00ff,#ff00ffff,#00ffffff,#7c0000ff,#00007cff,#007c00ff,#7c007cff,#007c7cff,#7c7c00ff" show_icon_for_attributes="true" show_tags="UNDER_NODES" show_note_icons="true" fit_to_viewport="false" show_icons="BESIDE_NODES" showTagCategories="false"/>
+    <tags category_separator="::"/>
+
+<map_styles>
+<stylenode LOCALIZED_TEXT="styles.root_node" STYLE="oval" UNIFORM_SHAPE="true" VGAP_QUANTITY="24 pt">
+<font SIZE="24"/>
+<stylenode LOCALIZED_TEXT="styles.predefined" POSITION="bottom_or_right" STYLE="bubble">
+<stylenode LOCALIZED_TEXT="default" ID="ID_271890427" ICON_SIZE="12 pt" COLOR="#000000" STYLE="fork">
+<arrowlink SHAPE="CUBIC_CURVE" COLOR="#000000" WIDTH="2" TRANSPARENCY="200" DASH="" FONT_SIZE="9" FONT_FAMILY="SansSerif" DESTINATION="ID_271890427" STARTARROW="NONE" ENDARROW="DEFAULT"/>
+<font NAME="SansSerif" SIZE="10" BOLD="false" ITALIC="false"/>
+<richcontent TYPE="DETAILS" CONTENT-TYPE="plain/auto"/>
+<richcontent TYPE="NOTE" CONTENT-TYPE="plain/auto"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="defaultstyle.details"/>
+<stylenode LOCALIZED_TEXT="defaultstyle.tags">
+<font SIZE="10"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="defaultstyle.attributes">
+<font SIZE="9"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="defaultstyle.note" COLOR="#000000" BACKGROUND_COLOR="#ffffff" TEXT_ALIGN="LEFT"/>
+<stylenode LOCALIZED_TEXT="defaultstyle.floating">
+<edge STYLE="hide_edge"/>
+<cloud COLOR="#f0f0f0" SHAPE="ROUND_RECT"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="defaultstyle.selection" BACKGROUND_COLOR="#afd3f7" BORDER_COLOR_LIKE_EDGE="false" BORDER_COLOR="#afd3f7"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="styles.user-defined" POSITION="bottom_or_right" STYLE="bubble">
+<stylenode LOCALIZED_TEXT="styles.topic" COLOR="#18898b" STYLE="fork">
+<font NAME="Liberation Sans" SIZE="10" BOLD="true"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="styles.subtopic" COLOR="#cc3300" STYLE="fork">
+<font NAME="Liberation Sans" SIZE="10" BOLD="true"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="styles.subsubtopic" COLOR="#669900">
+<font NAME="Liberation Sans" SIZE="10" BOLD="true"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="styles.important" ID="ID_67550811">
+<icon BUILTIN="yes"/>
+<arrowlink COLOR="#003399" TRANSPARENCY="255" DESTINATION="ID_67550811"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="styles.flower" COLOR="#ffffff" BACKGROUND_COLOR="#255aba" STYLE="oval" TEXT_ALIGN="CENTER" BORDER_WIDTH_LIKE_EDGE="false" BORDER_WIDTH="22 pt" BORDER_COLOR_LIKE_EDGE="false" BORDER_COLOR="#f9d71c" BORDER_DASH_LIKE_EDGE="false" BORDER_DASH="CLOSE_DOTS" MAX_WIDTH="6 cm" MIN_WIDTH="3 cm"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="styles.AutomaticLayout" POSITION="bottom_or_right" STYLE="bubble">
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level.root" COLOR="#000000" STYLE="oval" SHAPE_HORIZONTAL_MARGIN="10 pt" SHAPE_VERTICAL_MARGIN="10 pt">
+<font SIZE="18"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,1" COLOR="#0033ff">
+<font SIZE="16"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,2" COLOR="#00b439">
+<font SIZE="14"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,3" COLOR="#990000">
+<font SIZE="12"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,4" COLOR="#111111">
+<font SIZE="10"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,5"/>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,6"/>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,7"/>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,8"/>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,9"/>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,10"/>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,11"/>
+</stylenode>
+</stylenode>
+</map_styles>
+</hook>
+<node TEXT="IDataPageStreamReader" POSITION="bottom_or_right" ID="24" CREATED="1748275449538" MODIFIED="1748275449538">
+<node TEXT="Zeigt auf den aktuellen Datenpunkt" ID="25" CREATED="1748275449538" MODIFIED="1748275449538"/>
+<node TEXT="Nummer des Datenpunkts" ID="28" CREATED="1748275449538" MODIFIED="1748275449538"/>
+<node TEXT="MoveNext springt zum nächsten Datenpunkt." ID="30" CREATED="1748275449538" MODIFIED="1748275449538"/>
+<node TEXT="Seek(Idx)-Funktion springt zum Datenpunkt am gegebenen Index, relativ zum Anfang des Datensatzes." ID="40" CREATED="1748275449538" MODIFIED="1748275539454">
+<hook NAME="AlwaysUnfoldedNode"/>
+<node TEXT="Liefert den Index des tatsächlich erreichten Datensatzes zurück." ID="42" CREATED="1748275449538" MODIFIED="1748275449538"/>
+</node>
+<node TEXT="Die Gesamtzahl der Datensätze kann NICHT geliefert werden." ID="44" CREATED="1748275449538" MODIFIED="1748275449538">
+<node TEXT="Dafür müssten alle Dateien geöffnet werden, das wollen wir nicht." ID="45" CREATED="1748275449538" MODIFIED="1748275449538"/>
+</node>
+</node>
+<node TEXT="Streaming API für extrem große Datenpakete" POSITION="bottom_or_right" ID="3" CREATED="1748275449538" MODIFIED="1748275449538">
+<node TEXT="Daten sind lokal abgespeichert" ID="4" CREATED="1748275449538" MODIFIED="1748275449538">
+<node TEXT="Alle in einem Verzeichnis" ID="5" CREATED="1748275449538" MODIFIED="1748275449538">
+<node TEXT="Namensschema nach Zeitraum sortiert." ID="31" CREATED="1748275449538" MODIFIED="1748275449538">
+<node TEXT="Monate" ID="32" CREATED="1748275449538" MODIFIED="1748275449538">
+<node TEXT="Dateiname: &quot;DatenSatzName_JJJJ_MM.Extension&quot;" ID="33" CREATED="1748275449538" MODIFIED="1748275449538"/>
+</node>
+<node TEXT="Jahre" ID="34" CREATED="1748275449538" MODIFIED="1748275449538">
+<node TEXT="Dateiname: &quot;DatenSatzName_JJJJ.Extension&quot;" ID="35" CREATED="1748275449539" MODIFIED="1748275449539"/>
+</node>
+<node TEXT="Komprimierte Dateien" ID="37" CREATED="1748275449539" MODIFIED="1748275449539">
+<node TEXT="Hier wird an die Extension ein &quot;_zip&quot; angehängt." ID="38" CREATED="1748275449539" MODIFIED="1748275449539"/>
+</node>
+</node>
+<node TEXT="Der gesamte Datensatz wird durch DatenSatzName und Extension identifiziert." ID="36" CREATED="1748275449539" MODIFIED="1748275449539">
+<node TEXT="Der Loader muss die richtigen Dateinamen selbst herausfinden. Sie können entweder nach Monaten oder Jahren sortiert sein und sie können zusätzlich komprimiert sein." ID="39" CREATED="1748275449539" MODIFIED="1748275449539"/>
+</node>
+</node>
+<node TEXT="Paging algorhytmus" ID="6" CREATED="1748275449539" MODIFIED="1748275449539">
+<node TEXT="Es wird eine Reader benötigt, der sequentiell die Daten einließt." ID="10" CREATED="1748275449539" MODIFIED="1748275449539">
+<node TEXT="Reader ließt immer die aktuelle Page und im Hintergrund die potentiell nächste Page ein." ID="11" CREATED="1748275449539" MODIFIED="1748275449539">
+<node TEXT="Möglich wenig Latenz beim Zugriff." ID="16" CREATED="1748275449539" MODIFIED="1748275449539"/>
+<node TEXT="Es werden immer nur n Pages im Speicher gehalten. Wobei n &gt;= 2 ist." ID="17" CREATED="1748275449539" MODIFIED="1748275449539"/>
+</node>
+<node TEXT="Dateien könnten extern aktualisiert worden sein." ID="19" CREATED="1748275449539" MODIFIED="1748275449539">
+<node TEXT="Die eingelesene Page muss dann verworfen werden." ID="20" CREATED="1748275449540" MODIFIED="1748275449540"/>
+</node>
+</node>
+<node TEXT="Jede Datei repräsentiert eine &quot;Page&quot;" ID="12" CREATED="1748275449540" MODIFIED="1748275449540">
+<node TEXT="Dateien werden also immer komplett eingelesen." ID="13" CREATED="1748275449540" MODIFIED="1748275449540"/>
+<node TEXT="Dateien müssen also nicht geblockt werden" ID="14" CREATED="1748275449540" MODIFIED="1748275449540">
+<node TEXT="Besser für Netzwerkverkehr und unterstützt Caching auf Betriebssystemebene." ID="15" CREATED="1748275449540" MODIFIED="1748275449540"/>
+</node>
+</node>
+</node>
+</node>
+<node TEXT="Es handelt sich um Sequenzen von gleichgroßen Datenpunkten, also um Records." ID="21" CREATED="1748275449540" MODIFIED="1748275449540">
+<node TEXT="Der Loader bekommt den Typ des Records als generischen Parameter T mit." ID="22" CREATED="1748275449540" MODIFIED="1748275449540"/>
+</node>
+</node>
+</node>
+</map>

common/Myc.ChartDataController.pas

@@ -4,6 +4,7 @@ interface
 
 uses
     System.SysUtils, System.Classes, System.Generics.Collections, System.StrUtils,
+    Myc.Futures,
     Myc.TickLoader, Myc.OHLCCache; // OHLCCache for TOHLC, IGenericCandleBuilder, TF_ constants
 
 type
@@ -79,7 +80,6 @@ begin
         FOHLCCacheList.Add(nil); // Add a nil item to the list
     end;
 
-    SetLength(FTickDataArray, 0);
     if FMemoLog <> nil then
         FMemoLog.Add('TChartDataController instance created. Base timeframes initialized.');
 end;
@@ -211,16 +211,15 @@ procedure TChartDataController.LoadTickDataSeries(const AFileName: string);
 begin
     ClearAllDataAndCaches();
     try
-        FTickDataArray := Myc.TickLoader.LoadTickDataSeries(AFileName);
+        FTickDataArray := Myc.TickLoader.LoadTickDataSeries(AFileName).WaitFor;
         if FMemoLog <> nil then
-            FMemoLog.Add(Format('DataController: Loaded %d tick records from series: %s',
+            FMemoLog.Add(Format('DataController: Loading tick records from series: %s',
-                [Length(FTickDataArray), AFileName]));
+                [AFileName]));
     except
         on E: Exception do
         begin
             if FMemoLog <> nil then
                 FMemoLog.Add(Format('DataController: Error loading tick data from %s - %s', [AFileName, E.Message]));
-            SetLength(FTickDataArray, 0);
         end;
     end;
 end;

common/Myc.TickLoader.pas

@@ -1,33 +1,42 @@
 unit Myc.TickLoader;
 
-(*------------------------------------------------------------------------------
+(* ------------------------------------------------------------------------------
-  Unit:         Myc.TickLoader
+  Unit:         Myc.TickLoader
-  Author:       Delphi Algo Trading (Assistant)
+  Author:       Delphi Algo Trading (Assistant)
-  Date:         May 22, 2025
+  Date:         May 27, 2025
   Description:
-    This unit provides functions to load tick data (TTickData) from
+  This unit provides functions to load tick data (TTickData) from
-    binary files. The data originates from a C# application
+  binary files. The data originates from a C# application
-    and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
+  and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
 
-    The files now follow the naming convention: Symbol_Year.tab.
+  The files follow the naming convention:
-    (Prefix has been removed from the convention).
+  For .tab files:      Symbol_Year_MM.tab OR Symbol_Year_MM.tab_zip
-    Loading occurs in two phases:
+  For .tab-live files: Symbol_Year_MM.tab-live (NEVER compressed)
-    1. All .tab files are processed chronologically. Unique ticks are loaded,
+  (MM represents the two-digit month, e.g., 01 for January, 12 for December)
-       and the timestamp of the latest tick across all .tab files is determined
-       (overallLastTabTickTime). Assumes timestamps are unique across all .tab files.
-    2. All .tab-live files (for years corresponding to found .tab files) are
-       processed. Ticks from a .tab-live file are only integrated if their
-       timestamp is strictly later than overallLastTabTickTime. Assumes such
-       ticks are globally unique.
 
-    If a .tab-live file results in no new data being added under these rules,
+  If both compressed (.tab_zip) and uncompressed (.tab) versions of a .tab
-    it is deleted after processing.
+  file exist, the compressed version is preferred. Files ending with _zip
+  are expected to be ZIP archives containing the actual data file
+  (e.g., Symbol_Year_MM.tab).
 
-    The loading function automatically detects subsequent yearly .tab files.
+  Loading occurs in two phases:
+  1. All .tab files (regular or _zip) are processed chronologically by
+  month and year. Unique ticks are loaded, and the timestamp of the
+  latest tick across all .tab files is determined (overallLastTabTickTime).
+  Assumes timestamps are unique across all .tab files.
+  2. All .tab-live files (which are never compressed), for months/years
+  corresponding to found .tab files, are processed. Ticks from a
+  .tab-live file are only integrated if their timestamp is strictly later
+  than overallLastTabTickTime. Assumes such ticks are globally unique.
+
+  If a .tab-live file results in no new data being added under these rules,
+  it is deleted after processing.
+
+  The loading function automatically detects subsequent monthly/yearly files.
 
   Main Function:
-    LoadTickDataSeries - Loads a series of tick data files.
+  LoadTickDataSeries - Loads a series of tick data files.
-------------------------------------------------------------------------------*)
+  ------------------------------------------------------------------------------ *)
 
 interface
 
@@ -35,7 +44,9 @@ uses
     System.SysUtils,
     System.Classes,
     System.Generics.Collections,
-    System.IOUtils;
+    System.IOUtils,
+    Myc.Futures, // Added for TFuture in ReadFileTickData
+    Myc.Signals; // Added for IMycState (dependency of TFuture)
 
 type
     TTickData = packed record
@@ -43,211 +54,400 @@ type
         Ask: Single;
         Bid: Single;
     end;
+
     PTTickData = ^TTickData;
 
-// TryParseFileName now expects filenames like "Symbol_Year" (or "Symbol_Part2_Year")
+    // TryParseFileName parses filenames like "Symbol_Year_MM.Extension" or "Symbol_Year_MM.Extension_zip"
-// PrefixValue will be returned as an empty string.
+    // to extract Path, Symbol, Year, and Month.
-function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue: Integer): Boolean;
+function TryParseFileName( const FileName: string;
-function LoadTickDataSeries(const FileName: string): TArray<TTickData>;
+    out PathValue, SymbolValue: string;
+    out YearValue, MonthValue: Integer ): Boolean;
+
+function LoadTickDataSeries(const FileName: string): TFuture<TArray<TTickData>>;
+
+// Asynchronously reads tick data from a single specified file.
+// Returns a TFuture that will provide an array of TTickData.
+function ReadFileTickData( var LoadGate: TLatch; const FileName: string ): TFuture<TArray<TTickData>>;
 
 implementation
 
-function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue: Integer): Boolean;
+uses
+    System.Zip,
+    // Myc.Futures, Myc.Signals already in interface uses for TFuture type visibility
+    Myc.TaskManager; // Added for the global TaskManager used by TFuture.Construct
+
+function TryParseFileName( const FileName: string;
+    out PathValue, SymbolValue: string;
+    out YearValue, MonthValue: Integer ): Boolean;
 var
-    fileNameOnly: string;
+    fileNameNoPath: string;
+    nameForParsing: string;
     parts: TArray<string>;
 begin
-    Result := False;
+    Result := False; // Default to failure
-    PathValue := TPath.GetDirectoryName(FileName);
+    PathValue := ''; // Initialize all out parameters
-    fileNameOnly := TPath.GetFileNameWithoutExtension(FileName); // Handles any extension for parsing basename
+    SymbolValue := '';
+    YearValue := 0;
+    MonthValue := 0;
 
-    parts := fileNameOnly.Split(['_']);
+    PathValue := TPath.GetDirectoryName( FileName );
+    fileNameNoPath := TPath.GetFileName( FileName );
 
-    // Expected format is Symbol_Year or SymbolPart1_SymbolPart2_Year, so at least 2 parts.
+    nameForParsing := fileNameNoPath;
-    if Length(parts) < 2 then
+    if nameForParsing.EndsWith( '_zip', True ) then
     begin
-        SymbolValue := ''; YearValue := 0; // Clear out params on minimum parts failure
+        nameForParsing := nameForParsing.Substring( 0, nameForParsing.Length - 4 );
-        Exit;
     end;
 
-    // Year is the last part.
+    nameForParsing := TPath.GetFileNameWithoutExtension( nameForParsing ); // e.g., "EURUSD_2023_01"
-    if not TryStrToInt(parts[High(parts)], YearValue) then
+
+    parts := nameForParsing.Split( ['_'] );
+
+    if Length( parts ) < 3 then // Need at least Symbol_Year_Month
     begin
-        SymbolValue := ''; // Clear SymbolValue as well on year parse failure
+        exit;
-        Exit;
     end;
 
-    // Symbol consists of all parts before the year part.
+    // Month is the last part.
-    // Copy all parts except the last one (which is the Year).
+    if not TryStrToInt( parts[High( parts )], MonthValue ) then
-    // The second parameter of Copy (for TArray<string>) is the starting index,
+    begin
-    // the third parameter is the count of elements to copy.
+        exit;
-    // We want to copy elements from index 0 up to (but not including) the last element.
+    end;
-    // So, the count is High(parts) (which is Length(parts) - 1).
+    if ( MonthValue < 1 ) or ( MonthValue > 12 ) then // Validate month range
-    if High(parts) > 0 then // Ensure there's at least one part for the symbol
+    begin
-        SymbolValue := string.Join('_', Copy(parts, 0, High(parts)))
+        MonthValue := 0; // Set to invalid if out of range
-    else if Length(parts) = 1 then // Edge case: File is "Year.tab" - this is invalid by new rule "Symbol_Year"
+        exit;
-         begin SymbolValue := ''; Exit; end // Or handle as Symbol = parts[0] if "Year" itself is a symbol
+    end;
-    else // Should not happen if Length(parts) < 2 already exited
-        SymbolValue := '';
 
+    // Year is the second to last part.
+    if not TryStrToInt( parts[High( parts ) - 1], YearValue ) then
+    begin
+        MonthValue := 0; // Also invalidate month if year parsing fails
+        exit;
+    end;
+
+    // Symbol consists of all parts before Year and Month. Count = Length(parts) - 2
+    SymbolValue := string.Join( '_', Copy( parts, 0, Length( parts ) - 2 ) );
+
+    if SymbolValue = '' then // Ensure symbol is not empty
+    begin
+        exit;
+    end;
 
     Result := True;
 end;
 
-function LoadTickDataSeries(const FileName: string): TArray<TTickData>;
+function LoadTickDataSeries(const FileName: string): TFuture<TArray<TTickData>>;
-var
+// Local type declaration for storing file information for phase 1 processing
-    tickList: TList<TTickData>;
+type
-    initialYear, currentTabYear, liveProcessingYear: Integer;
+    TPhase1FileEntry = record
-    pathName, symbolName: string; // prefixNameOut will be an empty out param
+        Path: string;
-    tabFileName, liveFileName: string;
+        Year: Integer;
-    overallLastTabTickTime: Double;
+        Month: Integer;
-    maxTabYearFound: Integer;
-
-    procedure ReadFileContentAndGetMaxTime(const AFileName: string;
-                                      out Records: TArray<TTickData>;
-                                      out ActualMaxTickTimeInFile: Double);
-    var
-        fs: TFileStream;
-        fileSize: Int64;
-        recordCount: Integer;
-        bytesRead: Integer;
-        iterTickData: TTickData;
-        currentFileMaxOA: Double;
-    begin
-        SetLength(Records, 0);
-        ActualMaxTickTimeInFile := 0.0;
-        currentFileMaxOA := 0.0;
-
-        if not TFile.Exists(AFileName) then // Added check for file existence
-        begin
-            // Silently exit or log if preferred, but don't try to open non-existent file.
-            Exit;
-        end;
-
-        fs := TFileStream.Create(AFileName, fmOpenRead or fmShareDenyWrite);
-        try
-            fileSize := fs.Size;
-            if fileSize = 0 then Exit;
-
-            if (fileSize mod SizeOf(TTickData)) <> 0 then
-                raise EReadError.CreateFmt('File %s: Corrupted. Size %d is not a multiple of record size %d.',
-                    [AFileName, fileSize, SizeOf(TTickData)]);
-
-            recordCount := fileSize div SizeOf(TTickData);
-            if recordCount > 0 then
-            begin
-                SetLength(Records, recordCount);
-                bytesRead := fs.Read(Records[0], fileSize);
-                if bytesRead <> fileSize then
-                    raise EReadError.CreateFmt('File %s: Read error. Expected to read %d bytes, but actually read %d bytes.',
-                        [AFileName, fileSize, bytesRead]);
-
-                for iterTickData in Records do
-                begin
-                    if iterTickData.OADateTime > currentFileMaxOA then
-                        currentFileMaxOA := iterTickData.OADateTime;
-                end;
-                ActualMaxTickTimeInFile := currentFileMaxOA;
-            end;
-        finally
-            fs.Free;
-        end;
     end;
+var
+    initialYear, initialMonth: Integer;
+    currentTabYear, currentTabMonth: Integer;
+    pathName, symbolName: string;
+    tabFileEntryRecord: TPhase1FileEntry; // Variable to construct record instances
+    loadedState: TState;
+    loadedFiles: TArray<TFuture<TArray<TTickData>>>;
+    liveData: TFuture<TArray<TTickData>>;
 
 begin // Start of LoadTickDataSeries
-    tickList := TList<TTickData>.Create;
+    if not TryParseFileName( FileName, pathName, symbolName, initialYear, initialMonth ) then
-    overallLastTabTickTime := 0.0;
+    begin
-    maxTabYearFound := -1;
+        raise EArgumentException.CreateFmt(
+            'Invalid initial file name format: %s. Expected Symbol_Year_MM.Extension or Symbol_Year_MM.Extension_zip', [FileName] );
+    end;
 
+    currentTabYear := initialYear;
+    currentTabMonth := initialMonth;
+    var tabBaseNameFormat: string;
+    var compressedTabPath, uncompressedTabPath: string;
+    var actualTabFileToLoad: string;
+    var maxTabYearFound := -1;
+    var maxTabMonthFound := -1;
+
+    var filesToLoadPhase1 := TList<TPhase1FileEntry>.Create; // Create list with the named record type
     try
-        // Initial FileName must be the path to the first .tab file (e.g., Symbol_Year.tab)
+        while True do
-        // prefixNameOut will be an out parameter from TryParseFileName but will be empty.
-        if not TryParseFileName(FileName, pathName, symbolName, initialYear) then
         begin
-            raise EArgumentException.CreateFmt('Invalid initial file name format: %s. Expected Symbol_Year.tab (or Symbol_Part2_Year.tab)', [FileName]);
+            tabBaseNameFormat := Format( '%s_%.4d_%.2d.tab', [symbolName, currentTabYear, currentTabMonth] );
-        end;
+            compressedTabPath := TPath.Combine( pathName, tabBaseNameFormat + '_zip' );
+            uncompressedTabPath := TPath.Combine( pathName, tabBaseNameFormat );
 
-        // --- PHASE 1: Load all .tab files and determine overallLastTabTickTime ---
+            actualTabFileToLoad := '';
-        currentTabYear := initialYear;
+            if TFile.Exists( compressedTabPath ) then
-        // Construct the first .tab file name using the parsed components (symbol, year).
-        // Prefix is no longer used in the format string.
-        tabFileName := TPath.Combine(pathName, Format('%s_%d.tab', [symbolName, currentTabYear]));
-
-        var tabFileRecords: TArray<TTickData>;
-        var maxTimeInThisTabFile: Double;
-
-        while TFile.Exists(tabFileName) do
-        begin
-            maxTabYearFound := currentTabYear;
-            ReadFileContentAndGetMaxTime(tabFileName, tabFileRecords, maxTimeInThisTabFile);
-
-            if Length(tabFileRecords) > 0 then
             begin
-                tickList.AddRange(tabFileRecords);
+                actualTabFileToLoad := compressedTabPath;
+            end
+            else if TFile.Exists( uncompressedTabPath ) then
+            begin
+                actualTabFileToLoad := uncompressedTabPath;
             end;
 
-            if maxTimeInThisTabFile > overallLastTabTickTime then
+            if actualTabFileToLoad = '' then
-                overallLastTabTickTime := maxTimeInThisTabFile;
+                break; // No more sequential .tab files
 
-            Inc(currentTabYear);
+            // Construct the record and add it to the list
-            // Construct next tab filename without prefix
+            tabFileEntryRecord.Path := actualTabFileToLoad;
-            tabFileName := TPath.Combine(pathName, Format('%s_%d.tab', [symbolName, currentTabYear]));
+            tabFileEntryRecord.Year := currentTabYear;
+            tabFileEntryRecord.Month := currentTabMonth;
+            filesToLoadPhase1.Add( tabFileEntryRecord );
+
+            // Advance to the next month
+            Inc( currentTabMonth );
+            if currentTabMonth > 12 then
+            begin
+                currentTabMonth := 1;
+                Inc( currentTabYear );
+            end;
         end;
 
-        // --- PHASE 2: Load .tab-live files ---
+        // Now load and process identified .tab files
-        if maxTabYearFound >= initialYear then
+        var loadedFileList := TList<TFuture<TArray<TTickData>>>.Create;
-        begin
+        var loadStates := TList<TState>.Create;
-            for liveProcessingYear := initialYear to maxTabYearFound do
+        try
+            var LoadGate: TLatch;
+
+            for var tabFileEntry: TPhase1FileEntry in filesToLoadPhase1 do // Iterate with the correct type
             begin
-                // Construct .tab-live filename without prefix
+                var data := ReadFileTickData( LoadGate, tabFileEntry.Path );
-                liveFileName := TPath.Combine(pathName, Format('%s_%d.tab-live', [symbolName, liveProcessingYear]));
 
-                if TFile.Exists(liveFileName) then
+                loadedFileList.Add( TFuture<TArray<TTickData>>.Create( data ) );
+                loadStates.Add( data.Done );
+
+                if tabFileEntry.Year > maxTabYearFound then
                 begin
-                    var liveFileRecords: TArray<TTickData>;
+                    maxTabYearFound := tabFileEntry.Year;
-                    var maxTimeInThisLiveFile: Double;
+                    maxTabMonthFound := tabFileEntry.Month;
-                    var ticksToActuallyAddFromLive: TList<TTickData>;
+                end
-                    var liveDataAddedThisFile: Boolean;
+                else if ( tabFileEntry.Year = maxTabYearFound ) and ( tabFileEntry.Month > maxTabMonthFound ) then
-                    var iterLiveTickData: TTickData;
+                begin
-
+                    maxTabMonthFound := tabFileEntry.Month;
-                    ReadFileContentAndGetMaxTime(liveFileName, liveFileRecords, maxTimeInThisLiveFile);
-                    liveDataAddedThisFile := False;
-
-                    if Length(liveFileRecords) > 0 then
-                    begin
-                        ticksToActuallyAddFromLive := TList<TTickData>.Create;
-                        try
-                            for iterLiveTickData in liveFileRecords do
-                            begin
-                                if iterLiveTickData.OADateTime > overallLastTabTickTime then
-                                begin
-                                    ticksToActuallyAddFromLive.Add(iterLiveTickData);
-                                    liveDataAddedThisFile := True;
-                                end;
-                            end;
-
-                            if ticksToActuallyAddFromLive.Count > 0 then
-                                tickList.AddRange(ticksToActuallyAddFromLive);
-                        finally
-                            ticksToActuallyAddFromLive.Free;
-                        end;
-                    end;
-
-                    if not liveDataAddedThisFile then
-                    begin
-                        try
-                            TFile.Delete(liveFileName);
-                        except
-                            on E: Exception do {/* Optional: Log error, e.g., using a global logger or passed logger */}
-                        end;
-                    end;
                 end;
             end;
+
+            var liveFileBaseName := Format( '%s_%d_%02d.tab-live', [symbolName, maxTabYearFound, maxTabMonthFound] );
+            liveData := ReadFileTickData( LoadGate, TPath.Combine( pathName, liveFileBaseName ) );
+            loadStates.Add( liveData.Done );
+            loadedFileList.Add( liveData );
+
+            loadedState := TState.All( loadStates.ToArray );
+            loadedFiles := loadedFileList.ToArray;
+        finally
+            loadStates.Free;
+            loadedFileList.Free;
+        end;
+    finally
+        filesToLoadPhase1.Free;
+    end;
+
+    Result := TFuture<TArray<TTickData>>.Construct(
+        loadedState,
+        function: TArray<TTickData>
+        begin
+            var tickList := TList<TTickData>.Create;
+            try
+                var totalTicks := Length(liveData.Result);
+                var overallLastTabTickTime := 0.0;
+
+                for var P in loadedFiles do
+                    inc( totalTicks, Length(P.Result) );
+
+                tickList.Capacity := totalTicks;
+
+                for var P in loadedFiles do
+                begin
+                    if Length(P.Result) > 0 then
+                    begin
+                        inc( totalTicks, Length(P.Result) );
+
+                        for var iterTickData in P.Result do
+                            if iterTickData.OADateTime > overallLastTabTickTime then
+                            begin
+                                tickList.Add( iterTickData );
+                                overallLastTabTickTime := iterTickData.OADateTime;
+                            end;
+                    end;
+                end;
+
+                Result := tickList.ToArray;
+            finally
+                tickList.Free;
+            end;
+        end );
+end;
+
+function LoadSingleFileTickDataFromStream(const InputStream: TStream): TArray<TTickData>;
+var
+    fileSize: Int64;
+    recordCount: Integer;
+    bytesRead: Integer;
+begin
+    // Initialize Result record
+    SetLength(Result, 0);
+
+    InputStream.Position := 0;
+
+    fileSize := InputStream.Size;
+    if fileSize = 0 then
+        exit;
+
+    if (fileSize mod SizeOf(TTickData)) <> 0 then
+        raise EReadError.CreateFmt('File corrupted. Size %d is not a multiple of record size %d.',
+            [fileSize, SizeOf(TTickData)]);
+
+    recordCount := fileSize div SizeOf(TTickData);
+    if recordCount > 0 then
+    begin
+        SetLength(Result, recordCount);
+        bytesRead := InputStream.Read(Result[0], fileSize);
+        if bytesRead <> fileSize then
+            raise EReadError.CreateFmt('Read error. Expected to read %d bytes, but actually read %d bytes.',
+                [fileSize, bytesRead]);
+    end;
+end;
+
+function LoadUncompressedSingleFileTickData(const FullFileName: string): TArray<TTickData>;
+var
+    inputStream: TStream;
+begin
+    // Initialize Result record
+    SetLength(Result, 0);
+
+    if not TFile.Exists(FullFileName) then
+    begin
+        exit;
+    end;
+
+    inputStream := TFileStream.Create(FullFileName, fmOpenRead or fmShareDenyWrite);
+    try
+        try
+            Result := LoadSingleFileTickDataFromStream( inputStream );
+
+        except
+            on E: EReadError do
+            begin
+                raise EReadError.CreateFmt('File %s: %s', [FullFileName, E.Message]);
+            end;
+        end;
+    finally
+        inputStream.Free;
+    end;
+end;
+
+function UncompressTickDataFromStream(const InputStream: TStream): TArray<TTickData>;
+var
+    decompressionStream: TStream; // Holds uncompressed data of a single zip entry
+    localHeader: TZipHeader;
+    entryIndex: Integer;
+    i: Integer;
+    zipFileInstance: TZipFile;
+begin
+    // Initialize Result record
+    SetLength(Result, 0);
+
+    decompressionStream := nil;
+    zipFileInstance := nil;
+
+    try
+        InputStream.Position := 0; // Reset stream position to the beginning for TZipFile
+
+        // Open TZipFile from this memory stream
+        zipFileInstance := TZipFile.Create;
+        zipFileInstance.Open(InputStream, TZipMode.zmRead); // Open the zip from the memory stream
+
+        if zipFileInstance.FileCount = 0 then
+        begin
+            // If TZipFile opens an empty or invalid stream successfully but finds no files
+            exit; // FileSuccessfullyLoaded remains False
         end;
 
-        Result := tickList.ToArray;
+        entryIndex := -1;
+        for i := 0 to zipFileInstance.FileCount - 1 do
+        begin
+            if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
+            begin
+                entryIndex := i;
+                break;
+            end;
+        end;
+
+        if entryIndex = -1 then // If specific name not found, default to the first entry
+        begin
+            if zipFileInstance.FileCount > 0 then
+                entryIndex := 0
+            else
+                exit; // Should not happen if FileCount > 0 check passed
+        end;
+
+        // Decompress the specific entry from the in-memory zip file.
+        // TZipFile.Read creates and populates decompressionStream with uncompressed data.
+        zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
+
+        if not Assigned(decompressionStream) then
+            exit; // Exit if no valid stream could be prepared
+
+        Result := LoadSingleFileTickDataFromStream( decompressionStream );
     finally
-        tickList.Free;
+        // Ensure all potentially created objects are freed
+        if Assigned(decompressionStream) then // Holds uncompressed data from a zip entry
+            decompressionStream.Free;
+        if Assigned(zipFileInstance) then
+            zipFileInstance.Free;
+    end;
+end;
+
+function ReadFileTickData( var LoadGate: TLatch; const FileName: string ): TFuture<TArray<TTickData>>;
+var
+    parsedPath, parsedSymbol: string;
+    parsedYear, parsedMonth: Integer;
+begin
+    Result := TFuture<TArray<TTickData>>.Null;
+
+    // Attempt to parse year and month from the filename
+    if not TryParseFileName( FileName, parsedPath, parsedSymbol, parsedYear, parsedMonth ) then
+    begin
+        // If filename parsing fails, return a future that resolves to an empty array.
+        exit;
+    end;
+
+    if FileName.EndsWith('_zip', True) then
+    begin
+        var zipFileBytes := TFile.ReadAllBytes(FileName); // Read all bytes of the .zip file
+        if Length(zipFileBytes) = 0 then // Handle empty zip file case
+            exit;
+
+        var capFileName := FileName;
+        var blob := TFuture<TBytes>.Construct(
+            TLatch.Enqueue( LoadGate ),
+            function: TBytes
+            begin
+                Result := TFile.ReadAllBytes(capFileName); // Read all bytes of the .zip file
+            end );
+
+        Result := blob.Chain<TArray<TTickData>>(
+            function( bytes: TBytes ): TArray<TTickData>
+            begin
+                var zipMemoryStream := TBytesStream.Create( zipFileBytes ); // Create a memory stream to hold the zip file content
+                try
+                    zipMemoryStream.Position := 0;
+                    Result := UncompressTickDataFromStream( zipMemoryStream );
+                finally
+                    zipMemoryStream.Free;
+                end;
+            end );
+    end
+    else
+    begin
+        var capFileName := FileName;
+        Result := TFuture<TArray<TTickData>>.Construct(
+            function: TArray<TTickData>
+            begin
+                Result := LoadUncompressedSingleFileTickData( capFileName );
+            end );
     end;
 end;