diff --git a/ChartDisplay/ChartDisplayMain.dfm b/ChartDisplay/ChartDisplayMain.dfm
index d65acd2..5f75148 100644
--- a/ChartDisplay/ChartDisplayMain.dfm
+++ b/ChartDisplay/ChartDisplayMain.dfm
@@ -41,9 +41,7 @@ object ChartForm: TChartForm
Height = 23
Anchors = [akLeft, akTop, akRight]
TabOrder = 0
- Text =
- 'C:\Users\Brummel\Documents\cAlgo\TickData\Pepperstone\EURUSD_201' +
- '7.tab'
+ Text = '\\COFFEE\TickData\Pepperstone\EURUSD_2017_02.tab_zip'
end
object LoadButton: TButton
Left = 686
diff --git a/ChartDisplay/ChartDisplayMain.pas b/ChartDisplay/ChartDisplayMain.pas
index acc548f..a926e44 100644
--- a/ChartDisplay/ChartDisplayMain.pas
+++ b/ChartDisplay/ChartDisplayMain.pas
@@ -543,12 +543,13 @@ var
InitialDirectory: string;
SelectedPath, SelectedSymbol: string;
SelectedYear: Integer;
+ SelectedMonth: Integer;
IsValidSyntax: Boolean;
begin
OpenDialog := TOpenDialog.Create( Self );
try
OpenDialog.Title := 'Open Tick Data File';
- OpenDialog.Filter := 'Tick Data Files (*.tab)|*.TAB|All Files (*.*)|*.*';
+ OpenDialog.Filter := 'Tick Data Files (*.tab)|*.tab;*.tab_zip;*.tab_live|All Files (*.*)|*.*';
OpenDialog.DefaultExt := 'tab';
OpenDialog.Options := [ofFileMustExist, ofPathMustExist, ofHideReadOnly, ofEnableSizing];
@@ -574,11 +575,11 @@ begin
Memo.Lines.Add( FormatDateTime( 'yyyy-mm-dd hh:nn:ss', Now ) + ' - File selected via dialog: ' + OpenDialog.FileName );
IsValidSyntax := Myc.TickLoader.TryParseFileName( OpenDialog.FileName,
- SelectedPath, SelectedSymbol, SelectedYear );
+ SelectedPath, SelectedSymbol, SelectedYear, SelectedMonth );
if not IsValidSyntax then
begin
- MessageDlg( Format( 'Warning: The selected filename ''%s'' does not match the expected format (Prefix_Symbol_Year.tab).',
+ MessageDlg( Format( 'Warning: The selected filename ''%s'' does not match the expected format (Prefix_Symbol_Year_Month.tab).',
[System.IOUtils.TPath.GetFileName( OpenDialog.FileName )] ), mtWarning, [mbOK], 0 );
Memo.Lines.Add( 'Warning: Incorrect filename syntax chosen by user.' );
end;
diff --git a/ChartDisplay/ChartDisplayProject.dpr b/ChartDisplay/ChartDisplayProject.dpr
index 9d12255..2454eea 100644
--- a/ChartDisplay/ChartDisplayProject.dpr
+++ b/ChartDisplay/ChartDisplayProject.dpr
@@ -1,10 +1,12 @@
program ChartDisplayProject;
uses
+ FastMM5,
Vcl.Forms,
ChartDisplayMain in 'ChartDisplayMain.pas' {ChartForm},
Myc.OHLCCache in '..\common\Myc.OHLCCache.pas',
- Myc.ChartDataController in '..\common\Myc.ChartDataController.pas';
+ Myc.ChartDataController in '..\common\Myc.ChartDataController.pas',
+ DataStreamer in 'DataStreamer.pas';
{$R *.res}
diff --git a/ChartDisplay/ChartDisplayProject.dproj b/ChartDisplay/ChartDisplayProject.dproj
index 80778da..bbbc800 100644
--- a/ChartDisplay/ChartDisplayProject.dproj
+++ b/ChartDisplay/ChartDisplayProject.dproj
@@ -71,7 +71,7 @@
$(BDS)\bin\Artwork\Windows\UWP\delphi_UwpDefault_44.png
$(BDS)\bin\Artwork\Windows\UWP\delphi_UwpDefault_150.png
ChartDisplayProject
- ..\common;$(DCC_UnitSearchPath)
+ ..\common;T:\Myc\Src;$(DCC_UnitSearchPath)
1031
CompanyName=;FileDescription=$(MSBuildProjectName);FileVersion=1.0.0.0;InternalName=;LegalCopyright=;LegalTrademarks=;OriginalFilename=;ProgramID=com.embarcadero.$(MSBuildProjectName);ProductName=$(MSBuildProjectName);ProductVersion=1.0.0.0;Comments=
@@ -110,6 +110,9 @@
PerMonitorV2
+ true
+ true
+ 1033
false
@@ -129,10 +132,10 @@
- dfm
+
Base
@@ -153,12 +156,7 @@
ChartDisplayProject.dpr
-
- Embarcadero C++Builder Office 2000 Servers Package
- Embarcadero C++Builder Office XP Servers Package
- Microsoft Office 2000 Sample Automation Server Wrapper Components
- Microsoft Office XP Sample Automation Server Wrapper Components
-
+
@@ -173,9 +171,9 @@
true
-
+
- ChartDisplayProject.rsm
+ ChartDisplayProject.exe
true
diff --git a/ChartDisplay/DataStreamer.pas b/ChartDisplay/DataStreamer.pas
new file mode 100644
index 0000000..07f03ba
--- /dev/null
+++ b/ChartDisplay/DataStreamer.pas
@@ -0,0 +1,671 @@
+unit DataStreamer;
+
+interface
+
+uses
+ System.SysUtils, System.Classes, System.Generics.Collections, System.IOUtils, System.Threading,
+ System.Zip;
+
+type
+ IDataPage = interface
+ ['{B3E3C5B7-7C1A-4A8B-8B7E-15C9D9E0F4A6}']
+ function GetStream: TStream;
+ function GetSourceIdentifier: string;
+ function GetLastModificationTime: TDateTime;
+ end;
+
+ IRecordStreamReader = interface
+ ['{C1B7E8F0-5A0A-4F3B-BD28-18D5E5A3F7A0}']
+ function GetCurrentRecord: T;
+ function MoveToNextRecord: Boolean;
+ procedure Reset;
+ function GetCurrentRecordIndex: Int64;
+ function Seek(RecordIndex: Int64): Int64;
+
+ property CurrentRecord: T read GetCurrentRecord;
+ property CurrentRecordIndex: Int64 read GetCurrentRecordIndex;
+ end;
+
+ TLocalFilePage = class(TInterfacedObject, IDataPage)
+ private
+ FActualFilePath: string;
+ FMemoryStream: TMemoryStream;
+ FLastModifiedOnDisk: TDateTime;
+
+ function GetStream: TStream;
+ function GetSourceIdentifier: string;
+ function GetLastModificationTime: TDateTime;
+ public
+ constructor Create(const AActualFilePath: string; ADataStream: TMemoryStream; ALastModifiedTimeOnDisk: TDateTime);
+ destructor Destroy; override;
+ class function IsFileModified(const AFileName: string; KnownLastModified: TDateTime): Boolean;
+ end;
+
+ TStreamingFileRecordReader = class(TInterfacedObject, IRecordStreamReader)
+ private
+ FDirectoryPath: string;
+ FDataSetName: string;
+ FExtension: string;
+
+ FFilePaths: TStringList;
+ FCachedPages: TObjectDictionary;
+ FPageCacheQueue: TQueue;
+ FMaxCacheSize: Integer;
+ FPreloadPageTask: ITask;
+ FLockObject: TObject;
+
+ FCurrentPage: IDataPage;
+ FCurrentPageStream: TStream;
+ FCurrentPageIndexInFileList: Integer;
+
+ FRecordSize: Integer;
+ FCurrentRecord: T;
+ FRecordsInCurrentPage: Int64;
+ FRecordsReadFromCurrentPage: Int64;
+ FGlobalRecordIndex: Int64;
+
+ procedure ScanDirectoryAndMatchFiles;
+ function LoadPage(ActualFilePath: string): IDataPage;
+ procedure EnsurePageInCache(ActualFilePath: string; IsPreload: Boolean = False);
+ procedure EvictOldestPage;
+ procedure DoPreloadWork(const APathToPreload: string);
+ procedure StartPreloadNextPage;
+ procedure InvalidatePageIfModified(PageToCheck: IDataPage; IsCurrentPage: Boolean);
+ function ActivatePage(PageIndexInFileList: Integer): Boolean;
+ function GetRecordCountForFile(ActualFilePath: string): Int64;
+
+ function GetCurrentRecord: T;
+ function MoveToNextRecord: Boolean;
+ procedure Reset;
+ function GetCurrentRecordIndex: Int64;
+ function Seek(RecordIndex: Int64): Int64;
+ public
+ constructor Create(const ADirectoryPath: string; const ADataSetName: string; const AExtension: string; AMaxCacheSize: Integer = 2);
+ destructor Destroy; override;
+ end;
+
+implementation
+
+uses
+ System.DateUtils, System.Types, System.StrUtils,
+ System.Math;
+
+{ TLocalFilePage }
+
+constructor TLocalFilePage.Create(const AActualFilePath: string; ADataStream: TMemoryStream; ALastModifiedTimeOnDisk: TDateTime);
+begin
+ inherited Create;
+ FActualFilePath := AActualFilePath;
+ FMemoryStream := ADataStream;
+ FMemoryStream.Position := 0;
+ FLastModifiedOnDisk := ALastModifiedTimeOnDisk;
+end;
+
+destructor TLocalFilePage.Destroy;
+begin
+ FMemoryStream.Free;
+ inherited Destroy;
+end;
+
+function TLocalFilePage.GetStream: TStream;
+begin
+ FMemoryStream.Position := 0;
+ result := FMemoryStream;
+end;
+
+function TLocalFilePage.GetSourceIdentifier: string;
+begin
+ result := FActualFilePath;
+end;
+
+function TLocalFilePage.GetLastModificationTime: TDateTime;
+begin
+ result := FLastModifiedOnDisk;
+end;
+
+class function TLocalFilePage.IsFileModified(const AFileName: string; KnownLastModified: TDateTime): Boolean;
+var
+ currentModTime: TDateTime;
+begin
+ if TFile.Exists(AFileName) then
+ begin
+ currentModTime := TFile.GetLastWriteTime(AFileName);
+ result := Abs(MilliSecondsBetween(currentModTime, KnownLastModified)) > 1000;
+ end
+ else
+ begin
+ result := True;
+ end;
+end;
+
+{ TStreamingFileRecordReader }
+
+constructor TStreamingFileRecordReader.Create(const ADirectoryPath: string; const ADataSetName: string; const AExtension: string; AMaxCacheSize: Integer);
+begin
+ inherited Create;
+ FLockObject := TObject.Create;
+
+ if not TDirectory.Exists(ADirectoryPath) then
+ raise EDirectoryNotFoundException.Create('Directory not found: ' + ADirectoryPath);
+ if Trim(ADataSetName) = '' then
+ raise EArgumentException.Create('DataSetName cannot be empty.');
+ if Trim(AExtension) = '' then
+ raise EArgumentException.Create('Extension cannot be empty.');
+
+ FDirectoryPath := IncludeTrailingPathDelimiter(ADirectoryPath);
+ FDataSetName := ADataSetName;
+ FExtension := AExtension;
+ if StartsText('.', FExtension) then
+ FExtension := Copy(FExtension, 2, Length(FExtension) -1);
+
+ FFilePaths := TStringList.Create;
+ FFilePaths.Sorted := True;
+
+ FCachedPages := TObjectDictionary.Create([doOwnsValues]);
+ FPageCacheQueue := TQueue.Create;
+ FMaxCacheSize := System.Math.Max(2, AMaxCacheSize);
+ FRecordSize := SizeOf(T);
+ if FRecordSize = 0 then
+ raise EArgumentException.Create('Record size for type T cannot be zero.');
+
+ FCurrentPageIndexInFileList := -1;
+ FGlobalRecordIndex := -1;
+ FRecordsReadFromCurrentPage := 0;
+ FRecordsInCurrentPage := 0;
+ FCurrentPage := nil;
+ FCurrentPageStream := nil;
+
+ ScanDirectoryAndMatchFiles;
+ Reset;
+end;
+
+destructor TStreamingFileRecordReader.Destroy;
+var
+ taskToWaitFor: ITask;
+begin
+ taskToWaitFor := nil;
+ TMonitor.Enter(FLockObject);
+ try
+ taskToWaitFor := FPreloadPageTask;
+ FPreloadPageTask := nil;
+ finally
+ TMonitor.Exit(FLockObject);
+ end;
+
+ if (taskToWaitFor <> nil) and (taskToWaitFor.Status in [TTaskStatus.Created, TTaskStatus.WaitingToRun, TTaskStatus.Running]) then
+ begin
+ // Consider taskToWaitFor.Cancel; taskToWaitFor.Wait;
+ end;
+
+ FFilePaths.Free;
+ FCachedPages.Free;
+ FPageCacheQueue.Free;
+ FLockObject.Free;
+ inherited Destroy;
+end;
+
+procedure TStreamingFileRecordReader.ScanDirectoryAndMatchFiles;
+var
+ allFiles: TStringDynArray;
+ fileNameOnly, filePath: string;
+ expectedPrefix, baseExtensionPattern, zipExtensionPattern: string;
+begin
+ TMonitor.Enter(FLockObject);
+ try
+ FFilePaths.Clear;
+ allFiles := TDirectory.GetFiles(FDirectoryPath);
+ expectedPrefix := FDataSetName + '_';
+ baseExtensionPattern := '.' + FExtension;
+ zipExtensionPattern := baseExtensionPattern + '_zip';
+
+ for filePath in allFiles do
+ begin
+ fileNameOnly := ExtractFileName(filePath);
+ if StartsText(expectedPrefix, fileNameOnly) then
+ begin
+ if EndsText(zipExtensionPattern, fileNameOnly.ToLower) or EndsText(baseExtensionPattern, fileNameOnly.ToLower) then
+ begin
+ FFilePaths.Add(filePath);
+ end;
+ end;
+ end;
+ finally
+ TMonitor.Exit(FLockObject);
+ end;
+end;
+
+function TStreamingFileRecordReader.LoadPage(ActualFilePath: string): IDataPage;
+var
+ dataStream: TMemoryStream;
+ fileLastModified: TDateTime;
+ zipFile: TZipFile;
+ isCompressed: Boolean;
+ fileExtensionOnDisk: string;
+ decompressionStream: TStream;
+ localHeader: TZipHeader;
+ fileNameInZip: string;
+begin
+ dataStream := TMemoryStream.Create;
+ try
+ fileLastModified := TFile.GetLastWriteTime(ActualFilePath);
+ fileExtensionOnDisk := ExtractFileExt(ActualFilePath);
+ isCompressed := EndsText('_zip', fileExtensionOnDisk.ToLower);
+
+ if isCompressed then
+ begin
+ zipFile := TZipFile.Create;
+ try
+ zipFile.Open(ActualFilePath, zmRead);
+ if zipFile.FileCount > 0 then
+ begin
+ fileNameInZip := zipFile.FileNames[0];
+ zipFile.Read(fileNameInZip, decompressionStream, localHeader);
+ try
+ dataStream.CopyFrom(decompressionStream, 0);
+ finally
+ decompressionStream.Free;
+ end;
+ end
+ else
+ raise Exception.Create('Compressed file is empty: ' + ActualFilePath);
+ finally
+ zipFile.Free;
+ end;
+ end
+ else
+ dataStream.LoadFromFile(ActualFilePath);
+
+ dataStream.Position := 0;
+ result := TLocalFilePage.Create(ActualFilePath, dataStream, fileLastModified);
+ except
+ dataStream.Free;
+ raise;
+ end;
+end;
+
+procedure TStreamingFileRecordReader.EnsurePageInCache(ActualFilePath: string; IsPreload: Boolean);
+var
+ page: IDataPage;
+ newlyLoadedPage: IDataPage;
+begin
+ // Assumes FLockObject is held by the caller
+ if not FCachedPages.TryGetValue(ActualFilePath, page) then
+ begin
+ if FCachedPages.Count >= FMaxCacheSize then
+ EvictOldestPage;
+ newlyLoadedPage := LoadPage(ActualFilePath);
+ FCachedPages.Add(ActualFilePath, newlyLoadedPage);
+ FPageCacheQueue.Enqueue(ActualFilePath);
+ end
+ else
+ begin
+ if not IsPreload then
+ InvalidatePageIfModified(page, page = FCurrentPage);
+ end;
+end;
+
+procedure TStreamingFileRecordReader.EvictOldestPage;
+var
+ oldestFilePath: string;
+begin
+ // Assumes FLockObject is held by the caller
+ if FPageCacheQueue.Count > 0 then
+ begin
+ oldestFilePath := FPageCacheQueue.Dequeue;
+ if FCachedPages.ContainsKey(oldestFilePath) then
+ FCachedPages.Remove(oldestFilePath);
+ end;
+end;
+
+procedure TStreamingFileRecordReader.InvalidatePageIfModified(PageToCheck: IDataPage; IsCurrentPage: Boolean);
+var
+ actualFilePath: string;
+ reloadedPage: IDataPage;
+begin
+ // Assumes FLockObject is held by the caller
+ if PageToCheck = nil then exit;
+
+ actualFilePath := PageToCheck.GetSourceIdentifier;
+
+ if TLocalFilePage.IsFileModified(actualFilePath, PageToCheck.GetLastModificationTime) then
+ begin
+ FCachedPages.Remove(actualFilePath);
+
+ if IsCurrentPage then
+ begin
+ FCurrentPage := nil;
+ FCurrentPageStream := nil;
+
+ reloadedPage := LoadPage(actualFilePath);
+ FCachedPages.Add(actualFilePath, reloadedPage);
+
+ FCurrentPage := reloadedPage;
+ FCurrentPageStream := FCurrentPage.GetStream;
+ if (FCurrentPageStream <> nil) and (FCurrentPageStream.Size >= FRecordSize) then
+ FRecordsInCurrentPage := FCurrentPageStream.Size div FRecordSize
+ else
+ FRecordsInCurrentPage := 0;
+
+ if (FRecordsReadFromCurrentPage * FRecordSize < FCurrentPageStream.Size) and (FRecordsReadFromCurrentPage <= FRecordsInCurrentPage) then
+ FCurrentPageStream.Position := FRecordsReadFromCurrentPage * FRecordSize
+ else
+ begin
+ FCurrentPageStream.Position := 0;
+ FRecordsReadFromCurrentPage := 0;
+ end;
+ end;
+ end;
+end;
+
+procedure TStreamingFileRecordReader.DoPreloadWork(const APathToPreload: string);
+begin
+ try
+ TMonitor.Enter(FLockObject);
+ try
+ if not FCachedPages.ContainsKey(APathToPreload) then
+ EnsurePageInCache(APathToPreload, True);
+ finally
+ TMonitor.Exit(FLockObject);
+ end;
+ except
+ on E: Exception do
+ begin
+ // Optional: Log error E.Message
+ end;
+ end;
+end;
+
+procedure TStreamingFileRecordReader.StartPreloadNextPage;
+var
+ nextPageIndexToLoad: Integer;
+ nextFilePath: string;
+ currentTask: ITask;
+begin
+ TMonitor.Enter(FLockObject);
+ try
+ currentTask := FPreloadPageTask;
+ if (currentTask <> nil) and (currentTask.Status in [TTaskStatus.Created, TTaskStatus.WaitingToRun, TTaskStatus.Running]) then
+ exit;
+
+ nextPageIndexToLoad := FCurrentPageIndexInFileList + 1;
+ if nextPageIndexToLoad < FFilePaths.Count then
+ begin
+ nextFilePath := FFilePaths[nextPageIndexToLoad];
+ if FCachedPages.ContainsKey(nextFilePath) then
+ exit;
+
+ FPreloadPageTask := TTask.Run(
+ procedure
+ begin
+ DoPreloadWork(nextFilePath);
+ end
+ );
+ end;
+ finally
+ TMonitor.Exit(FLockObject);
+ end;
+end;
+
+function TStreamingFileRecordReader.ActivatePage(PageIndexInFileList: Integer): Boolean;
+var
+ actualFilePath: string;
+ pageFromCache: IDataPage;
+begin
+ result := False;
+ if (PageIndexInFileList < 0) or (PageIndexInFileList >= FFilePaths.Count) then
+ begin FCurrentPage := nil; FCurrentPageStream := nil; exit; end;
+
+ actualFilePath := FFilePaths[PageIndexInFileList];
+
+ TMonitor.Enter(FLockObject);
+ try
+ EnsurePageInCache(actualFilePath, False);
+ if not FCachedPages.TryGetValue(actualFilePath, pageFromCache) then
+ begin FCurrentPage := nil; FCurrentPageStream := nil; exit; end;
+
+ FCurrentPage := pageFromCache;
+ FCurrentPageStream := FCurrentPage.GetStream;
+ FCurrentPageIndexInFileList := PageIndexInFileList;
+ FRecordsReadFromCurrentPage := 0;
+ if (FCurrentPageStream <> nil) and (FCurrentPageStream.Size >= FRecordSize) then
+ FRecordsInCurrentPage := FCurrentPageStream.Size div FRecordSize
+ else
+ FRecordsInCurrentPage := 0;
+
+ if FRecordsInCurrentPage > 0 then
+ begin
+ result := True;
+ StartPreloadNextPage;
+ end;
+ finally
+ TMonitor.Exit(FLockObject);
+ end;
+end;
+
+function TStreamingFileRecordReader.GetRecordCountForFile(ActualFilePath: string): Int64;
+var
+ page: IDataPage;
+ stream: TStream;
+begin
+ result := 0;
+ EnsurePageInCache(ActualFilePath, False); // Assumes FLockObject is held by caller
+ if FCachedPages.TryGetValue(ActualFilePath, page) then
+ begin
+ stream := page.GetStream;
+ if (stream <> nil) and (stream.Size >= FRecordSize) then
+ result := stream.Size div FRecordSize;
+ end;
+end;
+
+function TStreamingFileRecordReader.GetCurrentRecord: T;
+begin
+ if FGlobalRecordIndex < 0 then
+ Result := System.Default(T)
+ else
+ Result := FCurrentRecord;
+end;
+
+function TStreamingFileRecordReader.GetCurrentRecordIndex: Int64;
+begin
+ result := FGlobalRecordIndex;
+end;
+
+function TStreamingFileRecordReader.MoveToNextRecord: Boolean;
+var
+ bytesRead: NativeInt; // ReadData returns NativeInt
+ attemptedNextPageActivation: Boolean;
+begin
+ result := False;
+ attemptedNextPageActivation := False;
+
+ TMonitor.Enter(FLockObject);
+ try
+ while True do
+ begin
+ if FCurrentPageStream = nil then
+ begin
+ if attemptedNextPageActivation or (FCurrentPageIndexInFileList >= FFilePaths.Count - 1) then
+ begin FGlobalRecordIndex := -1; exit; end;
+ if not ActivatePage(FCurrentPageIndexInFileList + 1) then
+ begin FGlobalRecordIndex := -1; exit; end;
+ attemptedNextPageActivation := True;
+ end;
+
+ if FRecordsReadFromCurrentPage >= FRecordsInCurrentPage then
+ begin
+ if attemptedNextPageActivation or (FCurrentPageIndexInFileList >= FFilePaths.Count - 1) then
+ begin FGlobalRecordIndex := -1; exit; end;
+ FCurrentPageStream := nil;
+ if not ActivatePage(FCurrentPageIndexInFileList + 1) then
+ begin FGlobalRecordIndex := -1; exit; end;
+ attemptedNextPageActivation := True;
+ continue;
+ end;
+
+ FCurrentPageStream.Position := FRecordsReadFromCurrentPage * FRecordSize;
+ if (FCurrentPageStream.Size - FCurrentPageStream.Position) < FRecordSize then
+ begin
+ FRecordsReadFromCurrentPage := FRecordsInCurrentPage;
+ attemptedNextPageActivation := False;
+ FCurrentPageStream := nil;
+ continue;
+ end;
+
+ // Verwende die generische TStream.ReadData Methode
+ bytesRead := FCurrentPageStream.ReadData(FCurrentRecord); // User line 518 context
+
+ if bytesRead = FRecordSize then // FRecordSize is SizeOf(T)
+ begin
+ Inc(FRecordsReadFromCurrentPage);
+ if FGlobalRecordIndex < 0 then
+ FGlobalRecordIndex := 0
+ else
+ Inc(FGlobalRecordIndex); // << User line 518 (E2010), hopefully fixed by ReadData
+ result := True; exit;
+ end
+ else
+ begin
+ FRecordsReadFromCurrentPage := FRecordsInCurrentPage;
+ attemptedNextPageActivation := False;
+ FCurrentPageStream := nil;
+ continue;
+ end;
+ end;
+ finally
+ TMonitor.Exit(FLockObject);
+ end;
+end;
+
+procedure TStreamingFileRecordReader.Reset;
+var
+ taskToCheck: ITask;
+begin
+ TMonitor.Enter(FLockObject);
+ try
+ taskToCheck := FPreloadPageTask;
+ if (taskToCheck <> nil) and (taskToCheck.Status in [TTaskStatus.Created, TTaskStatus.WaitingToRun, TTaskStatus.Running]) then
+ begin
+ // Cancellation logic
+ end;
+ FPreloadPageTask := nil;
+
+ FCurrentPageIndexInFileList := -1;
+ FGlobalRecordIndex := -1;
+ FRecordsReadFromCurrentPage := 0;
+ FRecordsInCurrentPage := 0;
+ FCurrentPage := nil;
+ FCurrentPageStream := nil;
+ FCurrentRecord := System.Default(T);
+ if FFilePaths.Count > 0 then
+ if not ActivatePage(0) then;
+ finally
+ TMonitor.Exit(FLockObject);
+ end;
+end;
+
+function TStreamingFileRecordReader.Seek(RecordIndex: Int64): Int64;
+var
+ pageIdx: Integer;
+ cumulativeRecordOffset: Int64;
+ recordsInCurrentFile: Int64;
+ recordOffsetInPage: Int64;
+ effectiveTargetIdx: Int64;
+ i: Integer;
+ sumRecordsBeforeThisPage: Int64;
+ bytesRead: NativeInt; // For ReadData
+begin
+ TMonitor.Enter(FLockObject);
+ try
+ if FFilePaths.Count = 0 then
+ begin
+ FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
+ FCurrentRecord := System.Default(T);
+ result := -1; exit;
+ end;
+
+ effectiveTargetIdx := RecordIndex;
+ if effectiveTargetIdx < 0 then effectiveTargetIdx := 0;
+
+ cumulativeRecordOffset := 0;
+ for pageIdx := 0 to FFilePaths.Count - 1 do
+ begin
+ recordsInCurrentFile := GetRecordCountForFile(FFilePaths[pageIdx]);
+ if effectiveTargetIdx < cumulativeRecordOffset + recordsInCurrentFile then
+ begin
+ if not ActivatePage(pageIdx) then
+ begin
+ FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
+ FCurrentRecord := System.Default(T);
+ result := -1; exit;
+ end;
+ recordOffsetInPage := effectiveTargetIdx - cumulativeRecordOffset;
+ FCurrentPageStream.Position := recordOffsetInPage * FRecordSize;
+
+ // Verwende die generische TStream.ReadData Methode
+ bytesRead := FCurrentPageStream.ReadData(FCurrentRecord);
+
+ if bytesRead = FRecordSize then
+ begin
+ FGlobalRecordIndex := effectiveTargetIdx; FRecordsReadFromCurrentPage := recordOffsetInPage + 1;
+ result := FGlobalRecordIndex; exit;
+ end
+ else
+ begin // User line ~604 context (E2008)
+ FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
+ FCurrentRecord := System.Default(T);
+ result := -1; exit;
+ end;
+ end;
+ inc(cumulativeRecordOffset, recordsInCurrentFile);
+ end;
+
+ if cumulativeRecordOffset = 0 then // User line ~616 context (E2029) for assignment after this block
+ begin
+ FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
+ FCurrentRecord := System.Default(T);
+ result := -1; exit;
+ end;
+
+ for pageIdx := FFilePaths.Count - 1 downto 0 do
+ begin
+ recordsInCurrentFile := GetRecordCountForFile(FFilePaths[pageIdx]);
+ if recordsInCurrentFile > 0 then
+ begin
+ if not ActivatePage(pageIdx) then
+ begin
+ FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
+ FCurrentRecord := System.Default(T);
+ result := -1; exit;
+ end;
+ recordOffsetInPage := recordsInCurrentFile - 1;
+ FCurrentPageStream.Position := recordOffsetInPage * FRecordSize;
+
+ // Verwende die generische TStream.ReadData Methode
+ bytesRead := FCurrentPageStream.ReadData(FCurrentRecord);
+
+ if bytesRead = FRecordSize then
+ begin
+ sumRecordsBeforeThisPage := 0;
+ for i := 0 to pageIdx - 1 do
+ sumRecordsBeforeThisPage := sumRecordsBeforeThisPage + GetRecordCountForFile(FFilePaths[i]);
+ FGlobalRecordIndex := sumRecordsBeforeThisPage + recordOffsetInPage; // User line 616 context (E2029/E2014)
+ FRecordsReadFromCurrentPage := recordOffsetInPage + 1;
+ result := FGlobalRecordIndex; exit;
+ end
+ else
+ begin // User line ~639 context (E2008)
+ FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
+ FCurrentRecord := System.Default(T);
+ result := -1; exit;
+ end;
+ end;
+ end;
+ FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
+ FCurrentRecord := System.Default(T);
+ result := -1; exit;
+ finally
+ TMonitor.Exit(FLockObject);
+ end;
+end;
+
+end.
diff --git a/DartaLoader.mm b/DartaLoader.mm
new file mode 100644
index 0000000..a38cdc3
--- /dev/null
+++ b/DartaLoader.mm
@@ -0,0 +1,127 @@
+
diff --git a/common/Myc.ChartDataController.pas b/common/Myc.ChartDataController.pas
index cbcef03..aff3f88 100644
--- a/common/Myc.ChartDataController.pas
+++ b/common/Myc.ChartDataController.pas
@@ -4,6 +4,7 @@ interface
uses
System.SysUtils, System.Classes, System.Generics.Collections, System.StrUtils,
+ Myc.Futures,
Myc.TickLoader, Myc.OHLCCache; // OHLCCache for TOHLC, IGenericCandleBuilder, TF_ constants
type
@@ -62,26 +63,25 @@ const
{ TChartDataController }
constructor TChartDataController.Create(AMemoTarget: TStrings);
-var
- I: Integer; // Standard loop variable
-begin
- inherited Create;
- FMemoLog := AMemoTarget;
- FGenericBuilders := TDictionary.Create;
-
- InitializeBaseTimeframeSettings; // This populates FTimeframeSettingsArray
-
- FOHLCCacheList := TList.Create; // Create the TList object
- // Add nil placeholders for each timeframe setting defined in FTimeframeSettingsArray
- // This ensures FOHLCCacheList is parallel to FTimeframeSettingsArray from the start.
- for I := 0 to High(FTimeframeSettingsArray) do
- begin
- FOHLCCacheList.Add(nil); // Add a nil item to the list
- end;
-
- SetLength(FTickDataArray, 0);
- if FMemoLog <> nil then
- FMemoLog.Add('TChartDataController instance created. Base timeframes initialized.');
+var
+ I: Integer; // Standard loop variable
+begin
+ inherited Create;
+ FMemoLog := AMemoTarget;
+ FGenericBuilders := TDictionary.Create;
+
+ InitializeBaseTimeframeSettings; // This populates FTimeframeSettingsArray
+
+ FOHLCCacheList := TList.Create; // Create the TList object
+ // Add nil placeholders for each timeframe setting defined in FTimeframeSettingsArray
+ // This ensures FOHLCCacheList is parallel to FTimeframeSettingsArray from the start.
+ for I := 0 to High(FTimeframeSettingsArray) do
+ begin
+ FOHLCCacheList.Add(nil); // Add a nil item to the list
+ end;
+
+ if FMemoLog <> nil then
+ FMemoLog.Add('TChartDataController instance created. Base timeframes initialized.');
end;
destructor TChartDataController.Destroy;
@@ -211,16 +211,15 @@ procedure TChartDataController.LoadTickDataSeries(const AFileName: string);
begin
ClearAllDataAndCaches();
try
- FTickDataArray := Myc.TickLoader.LoadTickDataSeries(AFileName);
+ FTickDataArray := Myc.TickLoader.LoadTickDataSeries(AFileName).WaitFor;
if FMemoLog <> nil then
- FMemoLog.Add(Format('DataController: Loaded %d tick records from series: %s',
- [Length(FTickDataArray), AFileName]));
+ FMemoLog.Add(Format('DataController: Loading tick records from series: %s',
+ [AFileName]));
except
on E: Exception do
begin
if FMemoLog <> nil then
FMemoLog.Add(Format('DataController: Error loading tick data from %s - %s', [AFileName, E.Message]));
- SetLength(FTickDataArray, 0);
end;
end;
end;
diff --git a/common/Myc.TickLoader.pas b/common/Myc.TickLoader.pas
index fb85cb2..6766a99 100644
--- a/common/Myc.TickLoader.pas
+++ b/common/Myc.TickLoader.pas
@@ -1,33 +1,42 @@
unit Myc.TickLoader;
-(*------------------------------------------------------------------------------
- Unit: Myc.TickLoader
- Author: Delphi Algo Trading (Assistant)
- Date: May 22, 2025
+(* ------------------------------------------------------------------------------
+ Unit: Myc.TickLoader
+ Author: Delphi Algo Trading (Assistant)
+ Date: May 27, 2025
Description:
- This unit provides functions to load tick data (TTickData) from
- binary files. The data originates from a C# application
- and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
+ This unit provides functions to load tick data (TTickData) from
+ binary files. The data originates from a C# application
+ and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
- The files now follow the naming convention: Symbol_Year.tab.
- (Prefix has been removed from the convention).
- Loading occurs in two phases:
- 1. All .tab files are processed chronologically. Unique ticks are loaded,
- and the timestamp of the latest tick across all .tab files is determined
- (overallLastTabTickTime). Assumes timestamps are unique across all .tab files.
- 2. All .tab-live files (for years corresponding to found .tab files) are
- processed. Ticks from a .tab-live file are only integrated if their
- timestamp is strictly later than overallLastTabTickTime. Assumes such
- ticks are globally unique.
+ The files follow the naming convention:
+ For .tab files: Symbol_Year_MM.tab OR Symbol_Year_MM.tab_zip
+ For .tab-live files: Symbol_Year_MM.tab-live (NEVER compressed)
+ (MM represents the two-digit month, e.g., 01 for January, 12 for December)
- If a .tab-live file results in no new data being added under these rules,
- it is deleted after processing.
+ If both compressed (.tab_zip) and uncompressed (.tab) versions of a .tab
+ file exist, the compressed version is preferred. Files ending with _zip
+ are expected to be ZIP archives containing the actual data file
+ (e.g., Symbol_Year_MM.tab).
- The loading function automatically detects subsequent yearly .tab files.
+ Loading occurs in two phases:
+ 1. All .tab files (regular or _zip) are processed chronologically by
+ month and year. Unique ticks are loaded, and the timestamp of the
+ latest tick across all .tab files is determined (overallLastTabTickTime).
+ Assumes timestamps are unique across all .tab files.
+ 2. All .tab-live files (which are never compressed), for months/years
+ corresponding to found .tab files, are processed. Ticks from a
+ .tab-live file are only integrated if their timestamp is strictly later
+ than overallLastTabTickTime. Assumes such ticks are globally unique.
+
+ If a .tab-live file results in no new data being added under these rules,
+ it is deleted after processing.
+
+ The loading function automatically detects subsequent monthly/yearly files.
Main Function:
- LoadTickDataSeries - Loads a series of tick data files.
-------------------------------------------------------------------------------*)
+ LoadTickDataSeries - Loads a series of tick data files.
+ ------------------------------------------------------------------------------ *)
interface
@@ -35,7 +44,9 @@ uses
System.SysUtils,
System.Classes,
System.Generics.Collections,
- System.IOUtils;
+ System.IOUtils,
+ Myc.Futures, // Added for TFuture in ReadFileTickData
+ Myc.Signals; // Added for IMycState (dependency of TFuture)
type
TTickData = packed record
@@ -43,211 +54,400 @@ type
Ask: Single;
Bid: Single;
end;
+
PTTickData = ^TTickData;
-// TryParseFileName now expects filenames like "Symbol_Year" (or "Symbol_Part2_Year")
-// PrefixValue will be returned as an empty string.
-function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue: Integer): Boolean;
-function LoadTickDataSeries(const FileName: string): TArray;
+ // TryParseFileName parses filenames like "Symbol_Year_MM.Extension" or "Symbol_Year_MM.Extension_zip"
+ // to extract Path, Symbol, Year, and Month.
+function TryParseFileName( const FileName: string;
+ out PathValue, SymbolValue: string;
+ out YearValue, MonthValue: Integer ): Boolean;
+
+function LoadTickDataSeries(const FileName: string): TFuture>;
+
+// Asynchronously reads tick data from a single specified file.
+// Returns a TFuture that will provide an array of TTickData.
+function ReadFileTickData( var LoadGate: TLatch; const FileName: string ): TFuture>;
implementation
-function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue: Integer): Boolean;
+uses
+ System.Zip,
+ // Myc.Futures, Myc.Signals already in interface uses for TFuture type visibility
+ Myc.TaskManager; // Added for the global TaskManager used by TFuture.Construct
+
+function TryParseFileName( const FileName: string;
+ out PathValue, SymbolValue: string;
+ out YearValue, MonthValue: Integer ): Boolean;
var
- fileNameOnly: string;
+ fileNameNoPath: string;
+ nameForParsing: string;
parts: TArray;
begin
- Result := False;
- PathValue := TPath.GetDirectoryName(FileName);
- fileNameOnly := TPath.GetFileNameWithoutExtension(FileName); // Handles any extension for parsing basename
+ Result := False; // Default to failure
+ PathValue := ''; // Initialize all out parameters
+ SymbolValue := '';
+ YearValue := 0;
+ MonthValue := 0;
- parts := fileNameOnly.Split(['_']);
+ PathValue := TPath.GetDirectoryName( FileName );
+ fileNameNoPath := TPath.GetFileName( FileName );
- // Expected format is Symbol_Year or SymbolPart1_SymbolPart2_Year, so at least 2 parts.
- if Length(parts) < 2 then
+ nameForParsing := fileNameNoPath;
+ if nameForParsing.EndsWith( '_zip', True ) then
begin
- SymbolValue := ''; YearValue := 0; // Clear out params on minimum parts failure
- Exit;
+ nameForParsing := nameForParsing.Substring( 0, nameForParsing.Length - 4 );
end;
- // Year is the last part.
- if not TryStrToInt(parts[High(parts)], YearValue) then
+ nameForParsing := TPath.GetFileNameWithoutExtension( nameForParsing ); // e.g., "EURUSD_2023_01"
+
+ parts := nameForParsing.Split( ['_'] );
+
+ if Length( parts ) < 3 then // Need at least Symbol_Year_Month
begin
- SymbolValue := ''; // Clear SymbolValue as well on year parse failure
- Exit;
+ exit;
end;
- // Symbol consists of all parts before the year part.
- // Copy all parts except the last one (which is the Year).
- // The second parameter of Copy (for TArray) is the starting index,
- // the third parameter is the count of elements to copy.
- // We want to copy elements from index 0 up to (but not including) the last element.
- // So, the count is High(parts) (which is Length(parts) - 1).
- if High(parts) > 0 then // Ensure there's at least one part for the symbol
- SymbolValue := string.Join('_', Copy(parts, 0, High(parts)))
- else if Length(parts) = 1 then // Edge case: File is "Year.tab" - this is invalid by new rule "Symbol_Year"
- begin SymbolValue := ''; Exit; end // Or handle as Symbol = parts[0] if "Year" itself is a symbol
- else // Should not happen if Length(parts) < 2 already exited
- SymbolValue := '';
+ // Month is the last part.
+ if not TryStrToInt( parts[High( parts )], MonthValue ) then
+ begin
+ exit;
+ end;
+ if ( MonthValue < 1 ) or ( MonthValue > 12 ) then // Validate month range
+ begin
+ MonthValue := 0; // Set to invalid if out of range
+ exit;
+ end;
+ // Year is the second to last part.
+ if not TryStrToInt( parts[High( parts ) - 1], YearValue ) then
+ begin
+ MonthValue := 0; // Also invalidate month if year parsing fails
+ exit;
+ end;
+
+ // Symbol consists of all parts before Year and Month. Count = Length(parts) - 2
+ SymbolValue := string.Join( '_', Copy( parts, 0, Length( parts ) - 2 ) );
+
+ if SymbolValue = '' then // Ensure symbol is not empty
+ begin
+ exit;
+ end;
Result := True;
end;
-function LoadTickDataSeries(const FileName: string): TArray;
-var
- tickList: TList;
- initialYear, currentTabYear, liveProcessingYear: Integer;
- pathName, symbolName: string; // prefixNameOut will be an empty out param
- tabFileName, liveFileName: string;
- overallLastTabTickTime: Double;
- maxTabYearFound: Integer;
-
- procedure ReadFileContentAndGetMaxTime(const AFileName: string;
- out Records: TArray;
- out ActualMaxTickTimeInFile: Double);
- var
- fs: TFileStream;
- fileSize: Int64;
- recordCount: Integer;
- bytesRead: Integer;
- iterTickData: TTickData;
- currentFileMaxOA: Double;
- begin
- SetLength(Records, 0);
- ActualMaxTickTimeInFile := 0.0;
- currentFileMaxOA := 0.0;
-
- if not TFile.Exists(AFileName) then // Added check for file existence
- begin
- // Silently exit or log if preferred, but don't try to open non-existent file.
- Exit;
- end;
-
- fs := TFileStream.Create(AFileName, fmOpenRead or fmShareDenyWrite);
- try
- fileSize := fs.Size;
- if fileSize = 0 then Exit;
-
- if (fileSize mod SizeOf(TTickData)) <> 0 then
- raise EReadError.CreateFmt('File %s: Corrupted. Size %d is not a multiple of record size %d.',
- [AFileName, fileSize, SizeOf(TTickData)]);
-
- recordCount := fileSize div SizeOf(TTickData);
- if recordCount > 0 then
- begin
- SetLength(Records, recordCount);
- bytesRead := fs.Read(Records[0], fileSize);
- if bytesRead <> fileSize then
- raise EReadError.CreateFmt('File %s: Read error. Expected to read %d bytes, but actually read %d bytes.',
- [AFileName, fileSize, bytesRead]);
-
- for iterTickData in Records do
- begin
- if iterTickData.OADateTime > currentFileMaxOA then
- currentFileMaxOA := iterTickData.OADateTime;
- end;
- ActualMaxTickTimeInFile := currentFileMaxOA;
- end;
- finally
- fs.Free;
- end;
+function LoadTickDataSeries(const FileName: string): TFuture>;
+// Local type declaration for storing file information for phase 1 processing
+type
+ TPhase1FileEntry = record
+ Path: string;
+ Year: Integer;
+ Month: Integer;
end;
+var
+ initialYear, initialMonth: Integer;
+ currentTabYear, currentTabMonth: Integer;
+ pathName, symbolName: string;
+ tabFileEntryRecord: TPhase1FileEntry; // Variable to construct record instances
+ loadedState: TState;
+ loadedFiles: TArray>>;
+ liveData: TFuture>;
begin // Start of LoadTickDataSeries
- tickList := TList.Create;
- overallLastTabTickTime := 0.0;
- maxTabYearFound := -1;
+ if not TryParseFileName( FileName, pathName, symbolName, initialYear, initialMonth ) then
+ begin
+ raise EArgumentException.CreateFmt(
+ 'Invalid initial file name format: %s. Expected Symbol_Year_MM.Extension or Symbol_Year_MM.Extension_zip', [FileName] );
+ end;
+ currentTabYear := initialYear;
+ currentTabMonth := initialMonth;
+ var tabBaseNameFormat: string;
+ var compressedTabPath, uncompressedTabPath: string;
+ var actualTabFileToLoad: string;
+ var maxTabYearFound := -1;
+ var maxTabMonthFound := -1;
+
+ var filesToLoadPhase1 := TList.Create; // Create list with the named record type
try
- // Initial FileName must be the path to the first .tab file (e.g., Symbol_Year.tab)
- // prefixNameOut will be an out parameter from TryParseFileName but will be empty.
- if not TryParseFileName(FileName, pathName, symbolName, initialYear) then
+ while True do
begin
- raise EArgumentException.CreateFmt('Invalid initial file name format: %s. Expected Symbol_Year.tab (or Symbol_Part2_Year.tab)', [FileName]);
- end;
+ tabBaseNameFormat := Format( '%s_%.4d_%.2d.tab', [symbolName, currentTabYear, currentTabMonth] );
+ compressedTabPath := TPath.Combine( pathName, tabBaseNameFormat + '_zip' );
+ uncompressedTabPath := TPath.Combine( pathName, tabBaseNameFormat );
- // --- PHASE 1: Load all .tab files and determine overallLastTabTickTime ---
- currentTabYear := initialYear;
- // Construct the first .tab file name using the parsed components (symbol, year).
- // Prefix is no longer used in the format string.
- tabFileName := TPath.Combine(pathName, Format('%s_%d.tab', [symbolName, currentTabYear]));
-
- var tabFileRecords: TArray;
- var maxTimeInThisTabFile: Double;
-
- while TFile.Exists(tabFileName) do
- begin
- maxTabYearFound := currentTabYear;
- ReadFileContentAndGetMaxTime(tabFileName, tabFileRecords, maxTimeInThisTabFile);
-
- if Length(tabFileRecords) > 0 then
+ actualTabFileToLoad := '';
+ if TFile.Exists( compressedTabPath ) then
begin
- tickList.AddRange(tabFileRecords);
+ actualTabFileToLoad := compressedTabPath;
+ end
+ else if TFile.Exists( uncompressedTabPath ) then
+ begin
+ actualTabFileToLoad := uncompressedTabPath;
end;
- if maxTimeInThisTabFile > overallLastTabTickTime then
- overallLastTabTickTime := maxTimeInThisTabFile;
+ if actualTabFileToLoad = '' then
+ break; // No more sequential .tab files
- Inc(currentTabYear);
- // Construct next tab filename without prefix
- tabFileName := TPath.Combine(pathName, Format('%s_%d.tab', [symbolName, currentTabYear]));
+ // Construct the record and add it to the list
+ tabFileEntryRecord.Path := actualTabFileToLoad;
+ tabFileEntryRecord.Year := currentTabYear;
+ tabFileEntryRecord.Month := currentTabMonth;
+ filesToLoadPhase1.Add( tabFileEntryRecord );
+
+ // Advance to the next month
+ Inc( currentTabMonth );
+ if currentTabMonth > 12 then
+ begin
+ currentTabMonth := 1;
+ Inc( currentTabYear );
+ end;
end;
- // --- PHASE 2: Load .tab-live files ---
- if maxTabYearFound >= initialYear then
- begin
- for liveProcessingYear := initialYear to maxTabYearFound do
+ // Now load and process identified .tab files
+ var loadedFileList := TList>>.Create;
+ var loadStates := TList.Create;
+ try
+ var LoadGate: TLatch;
+
+ for var tabFileEntry: TPhase1FileEntry in filesToLoadPhase1 do // Iterate with the correct type
begin
- // Construct .tab-live filename without prefix
- liveFileName := TPath.Combine(pathName, Format('%s_%d.tab-live', [symbolName, liveProcessingYear]));
+ var data := ReadFileTickData( LoadGate, tabFileEntry.Path );
- if TFile.Exists(liveFileName) then
+ loadedFileList.Add( TFuture>.Create( data ) );
+ loadStates.Add( data.Done );
+
+ if tabFileEntry.Year > maxTabYearFound then
begin
- var liveFileRecords: TArray;
- var maxTimeInThisLiveFile: Double;
- var ticksToActuallyAddFromLive: TList;
- var liveDataAddedThisFile: Boolean;
- var iterLiveTickData: TTickData;
-
- ReadFileContentAndGetMaxTime(liveFileName, liveFileRecords, maxTimeInThisLiveFile);
- liveDataAddedThisFile := False;
-
- if Length(liveFileRecords) > 0 then
- begin
- ticksToActuallyAddFromLive := TList.Create;
- try
- for iterLiveTickData in liveFileRecords do
- begin
- if iterLiveTickData.OADateTime > overallLastTabTickTime then
- begin
- ticksToActuallyAddFromLive.Add(iterLiveTickData);
- liveDataAddedThisFile := True;
- end;
- end;
-
- if ticksToActuallyAddFromLive.Count > 0 then
- tickList.AddRange(ticksToActuallyAddFromLive);
- finally
- ticksToActuallyAddFromLive.Free;
- end;
- end;
-
- if not liveDataAddedThisFile then
- begin
- try
- TFile.Delete(liveFileName);
- except
- on E: Exception do {/* Optional: Log error, e.g., using a global logger or passed logger */}
- end;
- end;
+ maxTabYearFound := tabFileEntry.Year;
+ maxTabMonthFound := tabFileEntry.Month;
+ end
+ else if ( tabFileEntry.Year = maxTabYearFound ) and ( tabFileEntry.Month > maxTabMonthFound ) then
+ begin
+ maxTabMonthFound := tabFileEntry.Month;
end;
end;
+
+ var liveFileBaseName := Format( '%s_%d_%02d.tab-live', [symbolName, maxTabYearFound, maxTabMonthFound] );
+ liveData := ReadFileTickData( LoadGate, TPath.Combine( pathName, liveFileBaseName ) );
+ loadStates.Add( liveData.Done );
+ loadedFileList.Add( liveData );
+
+ loadedState := TState.All( loadStates.ToArray );
+ loadedFiles := loadedFileList.ToArray;
+ finally
+ loadStates.Free;
+ loadedFileList.Free;
+ end;
+ finally
+ filesToLoadPhase1.Free;
+ end;
+
+ Result := TFuture>.Construct(
+ loadedState,
+ function: TArray
+ begin
+ var tickList := TList.Create;
+ try
+ var totalTicks := Length(liveData.Result);
+ var overallLastTabTickTime := 0.0;
+
+ for var P in loadedFiles do
+ inc( totalTicks, Length(P.Result) );
+
+ tickList.Capacity := totalTicks;
+
+ for var P in loadedFiles do
+ begin
+ if Length(P.Result) > 0 then
+ begin
+ inc( totalTicks, Length(P.Result) );
+
+ for var iterTickData in P.Result do
+ if iterTickData.OADateTime > overallLastTabTickTime then
+ begin
+ tickList.Add( iterTickData );
+ overallLastTabTickTime := iterTickData.OADateTime;
+ end;
+ end;
+ end;
+
+ Result := tickList.ToArray;
+ finally
+ tickList.Free;
+ end;
+ end );
+end;
+
+function LoadSingleFileTickDataFromStream(const InputStream: TStream): TArray;
+var
+ fileSize: Int64;
+ recordCount: Integer;
+ bytesRead: Integer;
+begin
+ // Initialize Result record
+ SetLength(Result, 0);
+
+ InputStream.Position := 0;
+
+ fileSize := InputStream.Size;
+ if fileSize = 0 then
+ exit;
+
+ if (fileSize mod SizeOf(TTickData)) <> 0 then
+ raise EReadError.CreateFmt('File corrupted. Size %d is not a multiple of record size %d.',
+ [fileSize, SizeOf(TTickData)]);
+
+ recordCount := fileSize div SizeOf(TTickData);
+ if recordCount > 0 then
+ begin
+ SetLength(Result, recordCount);
+ bytesRead := InputStream.Read(Result[0], fileSize);
+ if bytesRead <> fileSize then
+ raise EReadError.CreateFmt('Read error. Expected to read %d bytes, but actually read %d bytes.',
+ [fileSize, bytesRead]);
+ end;
+end;
+
+function LoadUncompressedSingleFileTickData(const FullFileName: string): TArray;
+var
+ inputStream: TStream;
+begin
+ // Initialize Result record
+ SetLength(Result, 0);
+
+ if not TFile.Exists(FullFileName) then
+ begin
+ exit;
+ end;
+
+ inputStream := TFileStream.Create(FullFileName, fmOpenRead or fmShareDenyWrite);
+ try
+ try
+ Result := LoadSingleFileTickDataFromStream( inputStream );
+
+ except
+ on E: EReadError do
+ begin
+ raise EReadError.CreateFmt('File %s: %s', [FullFileName, E.Message]);
+ end;
+ end;
+ finally
+ inputStream.Free;
+ end;
+end;
+
+function UncompressTickDataFromStream(const InputStream: TStream): TArray;
+var
+ decompressionStream: TStream; // Holds uncompressed data of a single zip entry
+ localHeader: TZipHeader;
+ entryIndex: Integer;
+ i: Integer;
+ zipFileInstance: TZipFile;
+begin
+ // Initialize Result record
+ SetLength(Result, 0);
+
+ decompressionStream := nil;
+ zipFileInstance := nil;
+
+ try
+ InputStream.Position := 0; // Reset stream position to the beginning for TZipFile
+
+ // Open TZipFile from this memory stream
+ zipFileInstance := TZipFile.Create;
+ zipFileInstance.Open(InputStream, TZipMode.zmRead); // Open the zip from the memory stream
+
+ if zipFileInstance.FileCount = 0 then
+ begin
+ // If TZipFile opens an empty or invalid stream successfully but finds no files
+ exit; // FileSuccessfullyLoaded remains False
end;
- Result := tickList.ToArray;
+ entryIndex := -1;
+ for i := 0 to zipFileInstance.FileCount - 1 do
+ begin
+ if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
+ begin
+ entryIndex := i;
+ break;
+ end;
+ end;
+
+ if entryIndex = -1 then // If specific name not found, default to the first entry
+ begin
+ if zipFileInstance.FileCount > 0 then
+ entryIndex := 0
+ else
+ exit; // Should not happen if FileCount > 0 check passed
+ end;
+
+ // Decompress the specific entry from the in-memory zip file.
+ // TZipFile.Read creates and populates decompressionStream with uncompressed data.
+ zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
+
+ if not Assigned(decompressionStream) then
+ exit; // Exit if no valid stream could be prepared
+
+ Result := LoadSingleFileTickDataFromStream( decompressionStream );
finally
- tickList.Free;
+ // Ensure all potentially created objects are freed
+ if Assigned(decompressionStream) then // Holds uncompressed data from a zip entry
+ decompressionStream.Free;
+ if Assigned(zipFileInstance) then
+ zipFileInstance.Free;
+ end;
+end;
+
+function ReadFileTickData( var LoadGate: TLatch; const FileName: string ): TFuture>;
+var
+ parsedPath, parsedSymbol: string;
+ parsedYear, parsedMonth: Integer;
+begin
+ Result := TFuture>.Null;
+
+ // Attempt to parse year and month from the filename
+ if not TryParseFileName( FileName, parsedPath, parsedSymbol, parsedYear, parsedMonth ) then
+ begin
+ // If filename parsing fails, return a future that resolves to an empty array.
+ exit;
+ end;
+
+ if FileName.EndsWith('_zip', True) then
+ begin
+ var zipFileBytes := TFile.ReadAllBytes(FileName); // Read all bytes of the .zip file
+ if Length(zipFileBytes) = 0 then // Handle empty zip file case
+ exit;
+
+ var capFileName := FileName;
+ var blob := TFuture.Construct(
+ TLatch.Enqueue( LoadGate ),
+ function: TBytes
+ begin
+ Result := TFile.ReadAllBytes(capFileName); // Read all bytes of the .zip file
+ end );
+
+ Result := blob.Chain>(
+ function( bytes: TBytes ): TArray
+ begin
+ var zipMemoryStream := TBytesStream.Create( zipFileBytes ); // Create a memory stream to hold the zip file content
+ try
+ zipMemoryStream.Position := 0;
+ Result := UncompressTickDataFromStream( zipMemoryStream );
+ finally
+ zipMemoryStream.Free;
+ end;
+ end );
+ end
+ else
+ begin
+ var capFileName := FileName;
+ Result := TFuture>.Construct(
+ function: TArray
+ begin
+ Result := LoadUncompressedSingleFileTickData( capFileName );
+ end );
end;
end;