integrated TFuture
This commit is contained in:
Michael Schimmel
@@ -41,9 +41,7 @@ object ChartForm: TChartForm
|
||||
Height = 23
|
||||
Anchors = [akLeft, akTop, akRight]
|
||||
TabOrder = 0
|
||||
Text =
|
||||
'C:\Users\Brummel\Documents\cAlgo\TickData\Pepperstone\EURUSD_201' +
|
||||
'7.tab'
|
||||
Text = '\\COFFEE\TickData\Pepperstone\EURUSD_2017_02.tab_zip'
|
||||
end
|
||||
object LoadButton: TButton
|
||||
Left = 686
|
||||
|
||||
@@ -543,12 +543,13 @@ var
|
||||
InitialDirectory: string;
|
||||
SelectedPath, SelectedSymbol: string;
|
||||
SelectedYear: Integer;
|
||||
SelectedMonth: Integer;
|
||||
IsValidSyntax: Boolean;
|
||||
begin
|
||||
OpenDialog := TOpenDialog.Create( Self );
|
||||
try
|
||||
OpenDialog.Title := 'Open Tick Data File';
|
||||
OpenDialog.Filter := 'Tick Data Files (*.tab)|*.TAB|All Files (*.*)|*.*';
|
||||
OpenDialog.Filter := 'Tick Data Files (*.tab)|*.tab;*.tab_zip;*.tab_live|All Files (*.*)|*.*';
|
||||
OpenDialog.DefaultExt := 'tab';
|
||||
OpenDialog.Options := [ofFileMustExist, ofPathMustExist, ofHideReadOnly, ofEnableSizing];
|
||||
|
||||
@@ -574,11 +575,11 @@ begin
|
||||
Memo.Lines.Add( FormatDateTime( 'yyyy-mm-dd hh:nn:ss', Now ) + ' - File selected via dialog: ' + OpenDialog.FileName );
|
||||
|
||||
IsValidSyntax := Myc.TickLoader.TryParseFileName( OpenDialog.FileName,
|
||||
SelectedPath, SelectedSymbol, SelectedYear );
|
||||
SelectedPath, SelectedSymbol, SelectedYear, SelectedMonth );
|
||||
|
||||
if not IsValidSyntax then
|
||||
begin
|
||||
MessageDlg( Format( 'Warning: The selected filename ''%s'' does not match the expected format (Prefix_Symbol_Year.tab).',
|
||||
MessageDlg( Format( 'Warning: The selected filename ''%s'' does not match the expected format (Prefix_Symbol_Year_Month.tab).',
|
||||
[System.IOUtils.TPath.GetFileName( OpenDialog.FileName )] ), mtWarning, [mbOK], 0 );
|
||||
Memo.Lines.Add( 'Warning: Incorrect filename syntax chosen by user.' );
|
||||
end;
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
program ChartDisplayProject;
|
||||
|
||||
uses
|
||||
FastMM5,
|
||||
Vcl.Forms,
|
||||
ChartDisplayMain in 'ChartDisplayMain.pas' {ChartForm},
|
||||
Myc.OHLCCache in '..\common\Myc.OHLCCache.pas',
|
||||
Myc.ChartDataController in '..\common\Myc.ChartDataController.pas';
|
||||
Myc.ChartDataController in '..\common\Myc.ChartDataController.pas',
|
||||
DataStreamer in 'DataStreamer.pas';
|
||||
|
||||
{$R *.res}
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@
|
||||
<UWP_DelphiLogo44>$(BDS)\bin\Artwork\Windows\UWP\delphi_UwpDefault_44.png</UWP_DelphiLogo44>
|
||||
<UWP_DelphiLogo150>$(BDS)\bin\Artwork\Windows\UWP\delphi_UwpDefault_150.png</UWP_DelphiLogo150>
|
||||
<SanitizedProjectName>ChartDisplayProject</SanitizedProjectName>
|
||||
<DCC_UnitSearchPath>..\common;$(DCC_UnitSearchPath)</DCC_UnitSearchPath>
|
||||
<DCC_UnitSearchPath>..\common;T:\Myc\Src;$(DCC_UnitSearchPath)</DCC_UnitSearchPath>
|
||||
<VerInfo_Locale>1031</VerInfo_Locale>
|
||||
<VerInfo_Keys>CompanyName=;FileDescription=$(MSBuildProjectName);FileVersion=1.0.0.0;InternalName=;LegalCopyright=;LegalTrademarks=;OriginalFilename=;ProgramID=com.embarcadero.$(MSBuildProjectName);ProductName=$(MSBuildProjectName);ProductVersion=1.0.0.0;Comments=</VerInfo_Keys>
|
||||
</PropertyGroup>
|
||||
@@ -110,6 +110,9 @@
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Cfg_1_Win64)'!=''">
|
||||
<AppDPIAwarenessMode>PerMonitorV2</AppDPIAwarenessMode>
|
||||
<DCC_RunTimeTypeInfo>true</DCC_RunTimeTypeInfo>
|
||||
<VerInfo_IncludeVerInfo>true</VerInfo_IncludeVerInfo>
|
||||
<VerInfo_Locale>1033</VerInfo_Locale>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Cfg_2)'!=''">
|
||||
<DCC_LocalDebugSymbols>false</DCC_LocalDebugSymbols>
|
||||
@@ -129,10 +132,10 @@
|
||||
</DelphiCompile>
|
||||
<DCCReference Include="ChartDisplayMain.pas">
|
||||
<Form>ChartForm</Form>
|
||||
<FormType>dfm</FormType>
|
||||
</DCCReference>
|
||||
<DCCReference Include="..\common\Myc.OHLCCache.pas"/>
|
||||
<DCCReference Include="..\common\Myc.ChartDataController.pas"/>
|
||||
<DCCReference Include="DataStreamer.pas"/>
|
||||
<BuildConfiguration Include="Base">
|
||||
<Key>Base</Key>
|
||||
</BuildConfiguration>
|
||||
@@ -153,12 +156,7 @@
|
||||
<Source>
|
||||
<Source Name="MainSource">ChartDisplayProject.dpr</Source>
|
||||
</Source>
|
||||
<Excluded_Packages>
|
||||
<Excluded_Packages Name="$(BDSBIN)\bcboffice2k290.bpl">Embarcadero C++Builder Office 2000 Servers Package</Excluded_Packages>
|
||||
<Excluded_Packages Name="$(BDSBIN)\bcbofficexp290.bpl">Embarcadero C++Builder Office XP Servers Package</Excluded_Packages>
|
||||
<Excluded_Packages Name="$(BDSBIN)\dcloffice2k290.bpl">Microsoft Office 2000 Sample Automation Server Wrapper Components</Excluded_Packages>
|
||||
<Excluded_Packages Name="$(BDSBIN)\dclofficexp290.bpl">Microsoft Office XP Sample Automation Server Wrapper Components</Excluded_Packages>
|
||||
</Excluded_Packages>
|
||||
<Excluded_Packages/>
|
||||
</Delphi.Personality>
|
||||
<Deployment Version="5">
|
||||
<DeployFile LocalName="Win32\Debug\ChartDisplayProject.exe" Configuration="Debug" Class="ProjectOutput">
|
||||
@@ -173,9 +171,9 @@
|
||||
<Overwrite>true</Overwrite>
|
||||
</Platform>
|
||||
</DeployFile>
|
||||
<DeployFile LocalName="Win64\Debug\ChartDisplayProject.rsm" Configuration="Debug" Class="DebugSymbols">
|
||||
<DeployFile LocalName="Win64\Release\ChartDisplayProject.exe" Configuration="Release" Class="ProjectOutput">
|
||||
<Platform Name="Win64">
|
||||
<RemoteName>ChartDisplayProject.rsm</RemoteName>
|
||||
<RemoteName>ChartDisplayProject.exe</RemoteName>
|
||||
<Overwrite>true</Overwrite>
|
||||
</Platform>
|
||||
</DeployFile>
|
||||
|
||||
@@ -0,0 +1,671 @@
|
||||
unit DataStreamer;
|
||||
|
||||
interface
|
||||
|
||||
uses
|
||||
System.SysUtils, System.Classes, System.Generics.Collections, System.IOUtils, System.Threading,
|
||||
System.Zip;
|
||||
|
||||
type
|
||||
IDataPage = interface
|
||||
['{B3E3C5B7-7C1A-4A8B-8B7E-15C9D9E0F4A6}']
|
||||
function GetStream: TStream;
|
||||
function GetSourceIdentifier: string;
|
||||
function GetLastModificationTime: TDateTime;
|
||||
end;
|
||||
|
||||
IRecordStreamReader<T: record> = interface
|
||||
['{C1B7E8F0-5A0A-4F3B-BD28-18D5E5A3F7A0}']
|
||||
function GetCurrentRecord: T;
|
||||
function MoveToNextRecord: Boolean;
|
||||
procedure Reset;
|
||||
function GetCurrentRecordIndex: Int64;
|
||||
function Seek(RecordIndex: Int64): Int64;
|
||||
|
||||
property CurrentRecord: T read GetCurrentRecord;
|
||||
property CurrentRecordIndex: Int64 read GetCurrentRecordIndex;
|
||||
end;
|
||||
|
||||
TLocalFilePage = class(TInterfacedObject, IDataPage)
|
||||
private
|
||||
FActualFilePath: string;
|
||||
FMemoryStream: TMemoryStream;
|
||||
FLastModifiedOnDisk: TDateTime;
|
||||
|
||||
function GetStream: TStream;
|
||||
function GetSourceIdentifier: string;
|
||||
function GetLastModificationTime: TDateTime;
|
||||
public
|
||||
constructor Create(const AActualFilePath: string; ADataStream: TMemoryStream; ALastModifiedTimeOnDisk: TDateTime);
|
||||
destructor Destroy; override;
|
||||
class function IsFileModified(const AFileName: string; KnownLastModified: TDateTime): Boolean;
|
||||
end;
|
||||
|
||||
TStreamingFileRecordReader<T: record> = class(TInterfacedObject, IRecordStreamReader<T>)
|
||||
private
|
||||
FDirectoryPath: string;
|
||||
FDataSetName: string;
|
||||
FExtension: string;
|
||||
|
||||
FFilePaths: TStringList;
|
||||
FCachedPages: TObjectDictionary<string, IDataPage>;
|
||||
FPageCacheQueue: TQueue<string>;
|
||||
FMaxCacheSize: Integer;
|
||||
FPreloadPageTask: ITask;
|
||||
FLockObject: TObject;
|
||||
|
||||
FCurrentPage: IDataPage;
|
||||
FCurrentPageStream: TStream;
|
||||
FCurrentPageIndexInFileList: Integer;
|
||||
|
||||
FRecordSize: Integer;
|
||||
FCurrentRecord: T;
|
||||
FRecordsInCurrentPage: Int64;
|
||||
FRecordsReadFromCurrentPage: Int64;
|
||||
FGlobalRecordIndex: Int64;
|
||||
|
||||
procedure ScanDirectoryAndMatchFiles;
|
||||
function LoadPage(ActualFilePath: string): IDataPage;
|
||||
procedure EnsurePageInCache(ActualFilePath: string; IsPreload: Boolean = False);
|
||||
procedure EvictOldestPage;
|
||||
procedure DoPreloadWork(const APathToPreload: string);
|
||||
procedure StartPreloadNextPage;
|
||||
procedure InvalidatePageIfModified(PageToCheck: IDataPage; IsCurrentPage: Boolean);
|
||||
function ActivatePage(PageIndexInFileList: Integer): Boolean;
|
||||
function GetRecordCountForFile(ActualFilePath: string): Int64;
|
||||
|
||||
function GetCurrentRecord: T;
|
||||
function MoveToNextRecord: Boolean;
|
||||
procedure Reset;
|
||||
function GetCurrentRecordIndex: Int64;
|
||||
function Seek(RecordIndex: Int64): Int64;
|
||||
public
|
||||
constructor Create(const ADirectoryPath: string; const ADataSetName: string; const AExtension: string; AMaxCacheSize: Integer = 2);
|
||||
destructor Destroy; override;
|
||||
end;
|
||||
|
||||
implementation
|
||||
|
||||
uses
|
||||
System.DateUtils, System.Types, System.StrUtils,
|
||||
System.Math;
|
||||
|
||||
{ TLocalFilePage }
|
||||
|
||||
constructor TLocalFilePage.Create(const AActualFilePath: string; ADataStream: TMemoryStream; ALastModifiedTimeOnDisk: TDateTime);
|
||||
begin
|
||||
inherited Create;
|
||||
FActualFilePath := AActualFilePath;
|
||||
FMemoryStream := ADataStream;
|
||||
FMemoryStream.Position := 0;
|
||||
FLastModifiedOnDisk := ALastModifiedTimeOnDisk;
|
||||
end;
|
||||
|
||||
destructor TLocalFilePage.Destroy;
|
||||
begin
|
||||
FMemoryStream.Free;
|
||||
inherited Destroy;
|
||||
end;
|
||||
|
||||
function TLocalFilePage.GetStream: TStream;
|
||||
begin
|
||||
FMemoryStream.Position := 0;
|
||||
result := FMemoryStream;
|
||||
end;
|
||||
|
||||
function TLocalFilePage.GetSourceIdentifier: string;
|
||||
begin
|
||||
result := FActualFilePath;
|
||||
end;
|
||||
|
||||
function TLocalFilePage.GetLastModificationTime: TDateTime;
|
||||
begin
|
||||
result := FLastModifiedOnDisk;
|
||||
end;
|
||||
|
||||
class function TLocalFilePage.IsFileModified(const AFileName: string; KnownLastModified: TDateTime): Boolean;
|
||||
var
|
||||
currentModTime: TDateTime;
|
||||
begin
|
||||
if TFile.Exists(AFileName) then
|
||||
begin
|
||||
currentModTime := TFile.GetLastWriteTime(AFileName);
|
||||
result := Abs(MilliSecondsBetween(currentModTime, KnownLastModified)) > 1000;
|
||||
end
|
||||
else
|
||||
begin
|
||||
result := True;
|
||||
end;
|
||||
end;
|
||||
|
||||
{ TStreamingFileRecordReader<T> }
|
||||
|
||||
constructor TStreamingFileRecordReader<T>.Create(const ADirectoryPath: string; const ADataSetName: string; const AExtension: string; AMaxCacheSize: Integer);
|
||||
begin
|
||||
inherited Create;
|
||||
FLockObject := TObject.Create;
|
||||
|
||||
if not TDirectory.Exists(ADirectoryPath) then
|
||||
raise EDirectoryNotFoundException.Create('Directory not found: ' + ADirectoryPath);
|
||||
if Trim(ADataSetName) = '' then
|
||||
raise EArgumentException.Create('DataSetName cannot be empty.');
|
||||
if Trim(AExtension) = '' then
|
||||
raise EArgumentException.Create('Extension cannot be empty.');
|
||||
|
||||
FDirectoryPath := IncludeTrailingPathDelimiter(ADirectoryPath);
|
||||
FDataSetName := ADataSetName;
|
||||
FExtension := AExtension;
|
||||
if StartsText('.', FExtension) then
|
||||
FExtension := Copy(FExtension, 2, Length(FExtension) -1);
|
||||
|
||||
FFilePaths := TStringList.Create;
|
||||
FFilePaths.Sorted := True;
|
||||
|
||||
FCachedPages := TObjectDictionary<string, IDataPage>.Create([doOwnsValues]);
|
||||
FPageCacheQueue := TQueue<string>.Create;
|
||||
FMaxCacheSize := System.Math.Max(2, AMaxCacheSize);
|
||||
FRecordSize := SizeOf(T);
|
||||
if FRecordSize = 0 then
|
||||
raise EArgumentException.Create('Record size for type T cannot be zero.');
|
||||
|
||||
FCurrentPageIndexInFileList := -1;
|
||||
FGlobalRecordIndex := -1;
|
||||
FRecordsReadFromCurrentPage := 0;
|
||||
FRecordsInCurrentPage := 0;
|
||||
FCurrentPage := nil;
|
||||
FCurrentPageStream := nil;
|
||||
|
||||
ScanDirectoryAndMatchFiles;
|
||||
Reset;
|
||||
end;
|
||||
|
||||
destructor TStreamingFileRecordReader<T>.Destroy;
|
||||
var
|
||||
taskToWaitFor: ITask;
|
||||
begin
|
||||
taskToWaitFor := nil;
|
||||
TMonitor.Enter(FLockObject);
|
||||
try
|
||||
taskToWaitFor := FPreloadPageTask;
|
||||
FPreloadPageTask := nil;
|
||||
finally
|
||||
TMonitor.Exit(FLockObject);
|
||||
end;
|
||||
|
||||
if (taskToWaitFor <> nil) and (taskToWaitFor.Status in [TTaskStatus.Created, TTaskStatus.WaitingToRun, TTaskStatus.Running]) then
|
||||
begin
|
||||
// Consider taskToWaitFor.Cancel; taskToWaitFor.Wait;
|
||||
end;
|
||||
|
||||
FFilePaths.Free;
|
||||
FCachedPages.Free;
|
||||
FPageCacheQueue.Free;
|
||||
FLockObject.Free;
|
||||
inherited Destroy;
|
||||
end;
|
||||
|
||||
procedure TStreamingFileRecordReader<T>.ScanDirectoryAndMatchFiles;
|
||||
var
|
||||
allFiles: TStringDynArray;
|
||||
fileNameOnly, filePath: string;
|
||||
expectedPrefix, baseExtensionPattern, zipExtensionPattern: string;
|
||||
begin
|
||||
TMonitor.Enter(FLockObject);
|
||||
try
|
||||
FFilePaths.Clear;
|
||||
allFiles := TDirectory.GetFiles(FDirectoryPath);
|
||||
expectedPrefix := FDataSetName + '_';
|
||||
baseExtensionPattern := '.' + FExtension;
|
||||
zipExtensionPattern := baseExtensionPattern + '_zip';
|
||||
|
||||
for filePath in allFiles do
|
||||
begin
|
||||
fileNameOnly := ExtractFileName(filePath);
|
||||
if StartsText(expectedPrefix, fileNameOnly) then
|
||||
begin
|
||||
if EndsText(zipExtensionPattern, fileNameOnly.ToLower) or EndsText(baseExtensionPattern, fileNameOnly.ToLower) then
|
||||
begin
|
||||
FFilePaths.Add(filePath);
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
finally
|
||||
TMonitor.Exit(FLockObject);
|
||||
end;
|
||||
end;
|
||||
|
||||
function TStreamingFileRecordReader<T>.LoadPage(ActualFilePath: string): IDataPage;
|
||||
var
|
||||
dataStream: TMemoryStream;
|
||||
fileLastModified: TDateTime;
|
||||
zipFile: TZipFile;
|
||||
isCompressed: Boolean;
|
||||
fileExtensionOnDisk: string;
|
||||
decompressionStream: TStream;
|
||||
localHeader: TZipHeader;
|
||||
fileNameInZip: string;
|
||||
begin
|
||||
dataStream := TMemoryStream.Create;
|
||||
try
|
||||
fileLastModified := TFile.GetLastWriteTime(ActualFilePath);
|
||||
fileExtensionOnDisk := ExtractFileExt(ActualFilePath);
|
||||
isCompressed := EndsText('_zip', fileExtensionOnDisk.ToLower);
|
||||
|
||||
if isCompressed then
|
||||
begin
|
||||
zipFile := TZipFile.Create;
|
||||
try
|
||||
zipFile.Open(ActualFilePath, zmRead);
|
||||
if zipFile.FileCount > 0 then
|
||||
begin
|
||||
fileNameInZip := zipFile.FileNames[0];
|
||||
zipFile.Read(fileNameInZip, decompressionStream, localHeader);
|
||||
try
|
||||
dataStream.CopyFrom(decompressionStream, 0);
|
||||
finally
|
||||
decompressionStream.Free;
|
||||
end;
|
||||
end
|
||||
else
|
||||
raise Exception.Create('Compressed file is empty: ' + ActualFilePath);
|
||||
finally
|
||||
zipFile.Free;
|
||||
end;
|
||||
end
|
||||
else
|
||||
dataStream.LoadFromFile(ActualFilePath);
|
||||
|
||||
dataStream.Position := 0;
|
||||
result := TLocalFilePage.Create(ActualFilePath, dataStream, fileLastModified);
|
||||
except
|
||||
dataStream.Free;
|
||||
raise;
|
||||
end;
|
||||
end;
|
||||
|
||||
procedure TStreamingFileRecordReader<T>.EnsurePageInCache(ActualFilePath: string; IsPreload: Boolean);
|
||||
var
|
||||
page: IDataPage;
|
||||
newlyLoadedPage: IDataPage;
|
||||
begin
|
||||
// Assumes FLockObject is held by the caller
|
||||
if not FCachedPages.TryGetValue(ActualFilePath, page) then
|
||||
begin
|
||||
if FCachedPages.Count >= FMaxCacheSize then
|
||||
EvictOldestPage;
|
||||
newlyLoadedPage := LoadPage(ActualFilePath);
|
||||
FCachedPages.Add(ActualFilePath, newlyLoadedPage);
|
||||
FPageCacheQueue.Enqueue(ActualFilePath);
|
||||
end
|
||||
else
|
||||
begin
|
||||
if not IsPreload then
|
||||
InvalidatePageIfModified(page, page = FCurrentPage);
|
||||
end;
|
||||
end;
|
||||
|
||||
procedure TStreamingFileRecordReader<T>.EvictOldestPage;
|
||||
var
|
||||
oldestFilePath: string;
|
||||
begin
|
||||
// Assumes FLockObject is held by the caller
|
||||
if FPageCacheQueue.Count > 0 then
|
||||
begin
|
||||
oldestFilePath := FPageCacheQueue.Dequeue;
|
||||
if FCachedPages.ContainsKey(oldestFilePath) then
|
||||
FCachedPages.Remove(oldestFilePath);
|
||||
end;
|
||||
end;
|
||||
|
||||
procedure TStreamingFileRecordReader<T>.InvalidatePageIfModified(PageToCheck: IDataPage; IsCurrentPage: Boolean);
|
||||
var
|
||||
actualFilePath: string;
|
||||
reloadedPage: IDataPage;
|
||||
begin
|
||||
// Assumes FLockObject is held by the caller
|
||||
if PageToCheck = nil then exit;
|
||||
|
||||
actualFilePath := PageToCheck.GetSourceIdentifier;
|
||||
|
||||
if TLocalFilePage.IsFileModified(actualFilePath, PageToCheck.GetLastModificationTime) then
|
||||
begin
|
||||
FCachedPages.Remove(actualFilePath);
|
||||
|
||||
if IsCurrentPage then
|
||||
begin
|
||||
FCurrentPage := nil;
|
||||
FCurrentPageStream := nil;
|
||||
|
||||
reloadedPage := LoadPage(actualFilePath);
|
||||
FCachedPages.Add(actualFilePath, reloadedPage);
|
||||
|
||||
FCurrentPage := reloadedPage;
|
||||
FCurrentPageStream := FCurrentPage.GetStream;
|
||||
if (FCurrentPageStream <> nil) and (FCurrentPageStream.Size >= FRecordSize) then
|
||||
FRecordsInCurrentPage := FCurrentPageStream.Size div FRecordSize
|
||||
else
|
||||
FRecordsInCurrentPage := 0;
|
||||
|
||||
if (FRecordsReadFromCurrentPage * FRecordSize < FCurrentPageStream.Size) and (FRecordsReadFromCurrentPage <= FRecordsInCurrentPage) then
|
||||
FCurrentPageStream.Position := FRecordsReadFromCurrentPage * FRecordSize
|
||||
else
|
||||
begin
|
||||
FCurrentPageStream.Position := 0;
|
||||
FRecordsReadFromCurrentPage := 0;
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
procedure TStreamingFileRecordReader<T>.DoPreloadWork(const APathToPreload: string);
|
||||
begin
|
||||
try
|
||||
TMonitor.Enter(FLockObject);
|
||||
try
|
||||
if not FCachedPages.ContainsKey(APathToPreload) then
|
||||
EnsurePageInCache(APathToPreload, True);
|
||||
finally
|
||||
TMonitor.Exit(FLockObject);
|
||||
end;
|
||||
except
|
||||
on E: Exception do
|
||||
begin
|
||||
// Optional: Log error E.Message
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
procedure TStreamingFileRecordReader<T>.StartPreloadNextPage;
|
||||
var
|
||||
nextPageIndexToLoad: Integer;
|
||||
nextFilePath: string;
|
||||
currentTask: ITask;
|
||||
begin
|
||||
TMonitor.Enter(FLockObject);
|
||||
try
|
||||
currentTask := FPreloadPageTask;
|
||||
if (currentTask <> nil) and (currentTask.Status in [TTaskStatus.Created, TTaskStatus.WaitingToRun, TTaskStatus.Running]) then
|
||||
exit;
|
||||
|
||||
nextPageIndexToLoad := FCurrentPageIndexInFileList + 1;
|
||||
if nextPageIndexToLoad < FFilePaths.Count then
|
||||
begin
|
||||
nextFilePath := FFilePaths[nextPageIndexToLoad];
|
||||
if FCachedPages.ContainsKey(nextFilePath) then
|
||||
exit;
|
||||
|
||||
FPreloadPageTask := TTask.Run(
|
||||
procedure
|
||||
begin
|
||||
DoPreloadWork(nextFilePath);
|
||||
end
|
||||
);
|
||||
end;
|
||||
finally
|
||||
TMonitor.Exit(FLockObject);
|
||||
end;
|
||||
end;
|
||||
|
||||
function TStreamingFileRecordReader<T>.ActivatePage(PageIndexInFileList: Integer): Boolean;
|
||||
var
|
||||
actualFilePath: string;
|
||||
pageFromCache: IDataPage;
|
||||
begin
|
||||
result := False;
|
||||
if (PageIndexInFileList < 0) or (PageIndexInFileList >= FFilePaths.Count) then
|
||||
begin FCurrentPage := nil; FCurrentPageStream := nil; exit; end;
|
||||
|
||||
actualFilePath := FFilePaths[PageIndexInFileList];
|
||||
|
||||
TMonitor.Enter(FLockObject);
|
||||
try
|
||||
EnsurePageInCache(actualFilePath, False);
|
||||
if not FCachedPages.TryGetValue(actualFilePath, pageFromCache) then
|
||||
begin FCurrentPage := nil; FCurrentPageStream := nil; exit; end;
|
||||
|
||||
FCurrentPage := pageFromCache;
|
||||
FCurrentPageStream := FCurrentPage.GetStream;
|
||||
FCurrentPageIndexInFileList := PageIndexInFileList;
|
||||
FRecordsReadFromCurrentPage := 0;
|
||||
if (FCurrentPageStream <> nil) and (FCurrentPageStream.Size >= FRecordSize) then
|
||||
FRecordsInCurrentPage := FCurrentPageStream.Size div FRecordSize
|
||||
else
|
||||
FRecordsInCurrentPage := 0;
|
||||
|
||||
if FRecordsInCurrentPage > 0 then
|
||||
begin
|
||||
result := True;
|
||||
StartPreloadNextPage;
|
||||
end;
|
||||
finally
|
||||
TMonitor.Exit(FLockObject);
|
||||
end;
|
||||
end;
|
||||
|
||||
function TStreamingFileRecordReader<T>.GetRecordCountForFile(ActualFilePath: string): Int64;
|
||||
var
|
||||
page: IDataPage;
|
||||
stream: TStream;
|
||||
begin
|
||||
result := 0;
|
||||
EnsurePageInCache(ActualFilePath, False); // Assumes FLockObject is held by caller
|
||||
if FCachedPages.TryGetValue(ActualFilePath, page) then
|
||||
begin
|
||||
stream := page.GetStream;
|
||||
if (stream <> nil) and (stream.Size >= FRecordSize) then
|
||||
result := stream.Size div FRecordSize;
|
||||
end;
|
||||
end;
|
||||
|
||||
function TStreamingFileRecordReader<T>.GetCurrentRecord: T;
|
||||
begin
|
||||
if FGlobalRecordIndex < 0 then
|
||||
Result := System.Default(T)
|
||||
else
|
||||
Result := FCurrentRecord;
|
||||
end;
|
||||
|
||||
function TStreamingFileRecordReader<T>.GetCurrentRecordIndex: Int64;
|
||||
begin
|
||||
result := FGlobalRecordIndex;
|
||||
end;
|
||||
|
||||
function TStreamingFileRecordReader<T>.MoveToNextRecord: Boolean;
|
||||
var
|
||||
bytesRead: NativeInt; // ReadData<T> returns NativeInt
|
||||
attemptedNextPageActivation: Boolean;
|
||||
begin
|
||||
result := False;
|
||||
attemptedNextPageActivation := False;
|
||||
|
||||
TMonitor.Enter(FLockObject);
|
||||
try
|
||||
while True do
|
||||
begin
|
||||
if FCurrentPageStream = nil then
|
||||
begin
|
||||
if attemptedNextPageActivation or (FCurrentPageIndexInFileList >= FFilePaths.Count - 1) then
|
||||
begin FGlobalRecordIndex := -1; exit; end;
|
||||
if not ActivatePage(FCurrentPageIndexInFileList + 1) then
|
||||
begin FGlobalRecordIndex := -1; exit; end;
|
||||
attemptedNextPageActivation := True;
|
||||
end;
|
||||
|
||||
if FRecordsReadFromCurrentPage >= FRecordsInCurrentPage then
|
||||
begin
|
||||
if attemptedNextPageActivation or (FCurrentPageIndexInFileList >= FFilePaths.Count - 1) then
|
||||
begin FGlobalRecordIndex := -1; exit; end;
|
||||
FCurrentPageStream := nil;
|
||||
if not ActivatePage(FCurrentPageIndexInFileList + 1) then
|
||||
begin FGlobalRecordIndex := -1; exit; end;
|
||||
attemptedNextPageActivation := True;
|
||||
continue;
|
||||
end;
|
||||
|
||||
FCurrentPageStream.Position := FRecordsReadFromCurrentPage * FRecordSize;
|
||||
if (FCurrentPageStream.Size - FCurrentPageStream.Position) < FRecordSize then
|
||||
begin
|
||||
FRecordsReadFromCurrentPage := FRecordsInCurrentPage;
|
||||
attemptedNextPageActivation := False;
|
||||
FCurrentPageStream := nil;
|
||||
continue;
|
||||
end;
|
||||
|
||||
// Verwende die generische TStream.ReadData<T> Methode
|
||||
bytesRead := FCurrentPageStream.ReadData<T>(FCurrentRecord); // User line 518 context
|
||||
|
||||
if bytesRead = FRecordSize then // FRecordSize is SizeOf(T)
|
||||
begin
|
||||
Inc(FRecordsReadFromCurrentPage);
|
||||
if FGlobalRecordIndex < 0 then
|
||||
FGlobalRecordIndex := 0
|
||||
else
|
||||
Inc(FGlobalRecordIndex); // << User line 518 (E2010), hopefully fixed by ReadData<T>
|
||||
result := True; exit;
|
||||
end
|
||||
else
|
||||
begin
|
||||
FRecordsReadFromCurrentPage := FRecordsInCurrentPage;
|
||||
attemptedNextPageActivation := False;
|
||||
FCurrentPageStream := nil;
|
||||
continue;
|
||||
end;
|
||||
end;
|
||||
finally
|
||||
TMonitor.Exit(FLockObject);
|
||||
end;
|
||||
end;
|
||||
|
||||
procedure TStreamingFileRecordReader<T>.Reset;
|
||||
var
|
||||
taskToCheck: ITask;
|
||||
begin
|
||||
TMonitor.Enter(FLockObject);
|
||||
try
|
||||
taskToCheck := FPreloadPageTask;
|
||||
if (taskToCheck <> nil) and (taskToCheck.Status in [TTaskStatus.Created, TTaskStatus.WaitingToRun, TTaskStatus.Running]) then
|
||||
begin
|
||||
// Cancellation logic
|
||||
end;
|
||||
FPreloadPageTask := nil;
|
||||
|
||||
FCurrentPageIndexInFileList := -1;
|
||||
FGlobalRecordIndex := -1;
|
||||
FRecordsReadFromCurrentPage := 0;
|
||||
FRecordsInCurrentPage := 0;
|
||||
FCurrentPage := nil;
|
||||
FCurrentPageStream := nil;
|
||||
FCurrentRecord := System.Default(T);
|
||||
if FFilePaths.Count > 0 then
|
||||
if not ActivatePage(0) then;
|
||||
finally
|
||||
TMonitor.Exit(FLockObject);
|
||||
end;
|
||||
end;
|
||||
|
||||
function TStreamingFileRecordReader<T>.Seek(RecordIndex: Int64): Int64;
|
||||
var
|
||||
pageIdx: Integer;
|
||||
cumulativeRecordOffset: Int64;
|
||||
recordsInCurrentFile: Int64;
|
||||
recordOffsetInPage: Int64;
|
||||
effectiveTargetIdx: Int64;
|
||||
i: Integer;
|
||||
sumRecordsBeforeThisPage: Int64;
|
||||
bytesRead: NativeInt; // For ReadData<T>
|
||||
begin
|
||||
TMonitor.Enter(FLockObject);
|
||||
try
|
||||
if FFilePaths.Count = 0 then
|
||||
begin
|
||||
FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
|
||||
FCurrentRecord := System.Default(T);
|
||||
result := -1; exit;
|
||||
end;
|
||||
|
||||
effectiveTargetIdx := RecordIndex;
|
||||
if effectiveTargetIdx < 0 then effectiveTargetIdx := 0;
|
||||
|
||||
cumulativeRecordOffset := 0;
|
||||
for pageIdx := 0 to FFilePaths.Count - 1 do
|
||||
begin
|
||||
recordsInCurrentFile := GetRecordCountForFile(FFilePaths[pageIdx]);
|
||||
if effectiveTargetIdx < cumulativeRecordOffset + recordsInCurrentFile then
|
||||
begin
|
||||
if not ActivatePage(pageIdx) then
|
||||
begin
|
||||
FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
|
||||
FCurrentRecord := System.Default(T);
|
||||
result := -1; exit;
|
||||
end;
|
||||
recordOffsetInPage := effectiveTargetIdx - cumulativeRecordOffset;
|
||||
FCurrentPageStream.Position := recordOffsetInPage * FRecordSize;
|
||||
|
||||
// Verwende die generische TStream.ReadData<T> Methode
|
||||
bytesRead := FCurrentPageStream.ReadData<T>(FCurrentRecord);
|
||||
|
||||
if bytesRead = FRecordSize then
|
||||
begin
|
||||
FGlobalRecordIndex := effectiveTargetIdx; FRecordsReadFromCurrentPage := recordOffsetInPage + 1;
|
||||
result := FGlobalRecordIndex; exit;
|
||||
end
|
||||
else
|
||||
begin // User line ~604 context (E2008)
|
||||
FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
|
||||
FCurrentRecord := System.Default(T);
|
||||
result := -1; exit;
|
||||
end;
|
||||
end;
|
||||
inc(cumulativeRecordOffset, recordsInCurrentFile);
|
||||
end;
|
||||
|
||||
if cumulativeRecordOffset = 0 then // User line ~616 context (E2029) for assignment after this block
|
||||
begin
|
||||
FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
|
||||
FCurrentRecord := System.Default(T);
|
||||
result := -1; exit;
|
||||
end;
|
||||
|
||||
for pageIdx := FFilePaths.Count - 1 downto 0 do
|
||||
begin
|
||||
recordsInCurrentFile := GetRecordCountForFile(FFilePaths[pageIdx]);
|
||||
if recordsInCurrentFile > 0 then
|
||||
begin
|
||||
if not ActivatePage(pageIdx) then
|
||||
begin
|
||||
FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
|
||||
FCurrentRecord := System.Default(T);
|
||||
result := -1; exit;
|
||||
end;
|
||||
recordOffsetInPage := recordsInCurrentFile - 1;
|
||||
FCurrentPageStream.Position := recordOffsetInPage * FRecordSize;
|
||||
|
||||
// Verwende die generische TStream.ReadData<T> Methode
|
||||
bytesRead := FCurrentPageStream.ReadData<T>(FCurrentRecord);
|
||||
|
||||
if bytesRead = FRecordSize then
|
||||
begin
|
||||
sumRecordsBeforeThisPage := 0;
|
||||
for i := 0 to pageIdx - 1 do
|
||||
sumRecordsBeforeThisPage := sumRecordsBeforeThisPage + GetRecordCountForFile(FFilePaths[i]);
|
||||
FGlobalRecordIndex := sumRecordsBeforeThisPage + recordOffsetInPage; // User line 616 context (E2029/E2014)
|
||||
FRecordsReadFromCurrentPage := recordOffsetInPage + 1;
|
||||
result := FGlobalRecordIndex; exit;
|
||||
end
|
||||
else
|
||||
begin // User line ~639 context (E2008)
|
||||
FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
|
||||
FCurrentRecord := System.Default(T);
|
||||
result := -1; exit;
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
FCurrentPageIndexInFileList := -1; FGlobalRecordIndex := -1; FCurrentPage := nil; FCurrentPageStream := nil;
|
||||
FCurrentRecord := System.Default(T);
|
||||
result := -1; exit;
|
||||
finally
|
||||
TMonitor.Exit(FLockObject);
|
||||
end;
|
||||
end;
|
||||
|
||||
end.
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
<map version="freeplane 1.12.1">
|
||||
<!--To view this file, download free mind mapping software Freeplane from https://www.freeplane.org -->
|
||||
<node TEXT="Data loader
" FOLDED="false" ID="1" CREATED="1748275449538" MODIFIED="1748275581071"><hook NAME="MapStyle">
|
||||
<properties edgeColorConfiguration="#808080ff,#ff0000ff,#0000ffff,#00ff00ff,#ff00ffff,#00ffffff,#7c0000ff,#00007cff,#007c00ff,#7c007cff,#007c7cff,#7c7c00ff" show_icon_for_attributes="true" show_tags="UNDER_NODES" show_note_icons="true" fit_to_viewport="false" show_icons="BESIDE_NODES" showTagCategories="false"/>
|
||||
<tags category_separator="::"/>
|
||||
|
||||
<map_styles>
|
||||
<stylenode LOCALIZED_TEXT="styles.root_node" STYLE="oval" UNIFORM_SHAPE="true" VGAP_QUANTITY="24 pt">
|
||||
<font SIZE="24"/>
|
||||
<stylenode LOCALIZED_TEXT="styles.predefined" POSITION="bottom_or_right" STYLE="bubble">
|
||||
<stylenode LOCALIZED_TEXT="default" ID="ID_271890427" ICON_SIZE="12 pt" COLOR="#000000" STYLE="fork">
|
||||
<arrowlink SHAPE="CUBIC_CURVE" COLOR="#000000" WIDTH="2" TRANSPARENCY="200" DASH="" FONT_SIZE="9" FONT_FAMILY="SansSerif" DESTINATION="ID_271890427" STARTARROW="NONE" ENDARROW="DEFAULT"/>
|
||||
<font NAME="SansSerif" SIZE="10" BOLD="false" ITALIC="false"/>
|
||||
<richcontent TYPE="DETAILS" CONTENT-TYPE="plain/auto"/>
|
||||
<richcontent TYPE="NOTE" CONTENT-TYPE="plain/auto"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="defaultstyle.details"/>
|
||||
<stylenode LOCALIZED_TEXT="defaultstyle.tags">
|
||||
<font SIZE="10"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="defaultstyle.attributes">
|
||||
<font SIZE="9"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="defaultstyle.note" COLOR="#000000" BACKGROUND_COLOR="#ffffff" TEXT_ALIGN="LEFT"/>
|
||||
<stylenode LOCALIZED_TEXT="defaultstyle.floating">
|
||||
<edge STYLE="hide_edge"/>
|
||||
<cloud COLOR="#f0f0f0" SHAPE="ROUND_RECT"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="defaultstyle.selection" BACKGROUND_COLOR="#afd3f7" BORDER_COLOR_LIKE_EDGE="false" BORDER_COLOR="#afd3f7"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="styles.user-defined" POSITION="bottom_or_right" STYLE="bubble">
|
||||
<stylenode LOCALIZED_TEXT="styles.topic" COLOR="#18898b" STYLE="fork">
|
||||
<font NAME="Liberation Sans" SIZE="10" BOLD="true"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="styles.subtopic" COLOR="#cc3300" STYLE="fork">
|
||||
<font NAME="Liberation Sans" SIZE="10" BOLD="true"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="styles.subsubtopic" COLOR="#669900">
|
||||
<font NAME="Liberation Sans" SIZE="10" BOLD="true"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="styles.important" ID="ID_67550811">
|
||||
<icon BUILTIN="yes"/>
|
||||
<arrowlink COLOR="#003399" TRANSPARENCY="255" DESTINATION="ID_67550811"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="styles.flower" COLOR="#ffffff" BACKGROUND_COLOR="#255aba" STYLE="oval" TEXT_ALIGN="CENTER" BORDER_WIDTH_LIKE_EDGE="false" BORDER_WIDTH="22 pt" BORDER_COLOR_LIKE_EDGE="false" BORDER_COLOR="#f9d71c" BORDER_DASH_LIKE_EDGE="false" BORDER_DASH="CLOSE_DOTS" MAX_WIDTH="6 cm" MIN_WIDTH="3 cm"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="styles.AutomaticLayout" POSITION="bottom_or_right" STYLE="bubble">
|
||||
<stylenode LOCALIZED_TEXT="AutomaticLayout.level.root" COLOR="#000000" STYLE="oval" SHAPE_HORIZONTAL_MARGIN="10 pt" SHAPE_VERTICAL_MARGIN="10 pt">
|
||||
<font SIZE="18"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="AutomaticLayout.level,1" COLOR="#0033ff">
|
||||
<font SIZE="16"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="AutomaticLayout.level,2" COLOR="#00b439">
|
||||
<font SIZE="14"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="AutomaticLayout.level,3" COLOR="#990000">
|
||||
<font SIZE="12"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="AutomaticLayout.level,4" COLOR="#111111">
|
||||
<font SIZE="10"/>
|
||||
</stylenode>
|
||||
<stylenode LOCALIZED_TEXT="AutomaticLayout.level,5"/>
|
||||
<stylenode LOCALIZED_TEXT="AutomaticLayout.level,6"/>
|
||||
<stylenode LOCALIZED_TEXT="AutomaticLayout.level,7"/>
|
||||
<stylenode LOCALIZED_TEXT="AutomaticLayout.level,8"/>
|
||||
<stylenode LOCALIZED_TEXT="AutomaticLayout.level,9"/>
|
||||
<stylenode LOCALIZED_TEXT="AutomaticLayout.level,10"/>
|
||||
<stylenode LOCALIZED_TEXT="AutomaticLayout.level,11"/>
|
||||
</stylenode>
|
||||
</stylenode>
|
||||
</map_styles>
|
||||
</hook>
|
||||
<node TEXT="IDataPageStreamReader" POSITION="bottom_or_right" ID="24" CREATED="1748275449538" MODIFIED="1748275449538">
|
||||
<node TEXT="Zeigt auf den aktuellen Datenpunkt" ID="25" CREATED="1748275449538" MODIFIED="1748275449538"/>
|
||||
<node TEXT="Nummer des Datenpunkts" ID="28" CREATED="1748275449538" MODIFIED="1748275449538"/>
|
||||
<node TEXT="MoveNext springt zum nächsten Datenpunkt." ID="30" CREATED="1748275449538" MODIFIED="1748275449538"/>
|
||||
<node TEXT="Seek(Idx)-Funktion springt zum Datenpunkt am gegebenen Index, relativ zum Anfang des Datensatzes." ID="40" CREATED="1748275449538" MODIFIED="1748275539454">
|
||||
<hook NAME="AlwaysUnfoldedNode"/>
|
||||
<node TEXT="Liefert den Index des tatsächlich erreichten Datensatzes zurück." ID="42" CREATED="1748275449538" MODIFIED="1748275449538"/>
|
||||
</node>
|
||||
<node TEXT="Die Gesamtzahl der Datensätze kann NICHT geliefert werden." ID="44" CREATED="1748275449538" MODIFIED="1748275449538">
|
||||
<node TEXT="Dafür müssten alle Dateien geöffnet werden, das wollen wir nicht." ID="45" CREATED="1748275449538" MODIFIED="1748275449538"/>
|
||||
</node>
|
||||
</node>
|
||||
<node TEXT="Streaming API für extrem große Datenpakete" POSITION="bottom_or_right" ID="3" CREATED="1748275449538" MODIFIED="1748275449538">
|
||||
<node TEXT="Daten sind lokal abgespeichert" ID="4" CREATED="1748275449538" MODIFIED="1748275449538">
|
||||
<node TEXT="Alle in einem Verzeichnis" ID="5" CREATED="1748275449538" MODIFIED="1748275449538">
|
||||
<node TEXT="Namensschema nach Zeitraum sortiert." ID="31" CREATED="1748275449538" MODIFIED="1748275449538">
|
||||
<node TEXT="Monate" ID="32" CREATED="1748275449538" MODIFIED="1748275449538">
|
||||
<node TEXT="Dateiname: "DatenSatzName_JJJJ_MM.Extension"" ID="33" CREATED="1748275449538" MODIFIED="1748275449538"/>
|
||||
</node>
|
||||
<node TEXT="Jahre" ID="34" CREATED="1748275449538" MODIFIED="1748275449538">
|
||||
<node TEXT="Dateiname: "DatenSatzName_JJJJ.Extension"" ID="35" CREATED="1748275449539" MODIFIED="1748275449539"/>
|
||||
</node>
|
||||
<node TEXT="Komprimierte Dateien" ID="37" CREATED="1748275449539" MODIFIED="1748275449539">
|
||||
<node TEXT="Hier wird an die Extension ein "_zip" angehängt." ID="38" CREATED="1748275449539" MODIFIED="1748275449539"/>
|
||||
</node>
|
||||
</node>
|
||||
<node TEXT="Der gesamte Datensatz wird durch DatenSatzName und Extension identifiziert." ID="36" CREATED="1748275449539" MODIFIED="1748275449539">
|
||||
<node TEXT="Der Loader muss die richtigen Dateinamen selbst herausfinden. Sie können entweder nach Monaten oder Jahren sortiert sein und sie können zusätzlich komprimiert sein." ID="39" CREATED="1748275449539" MODIFIED="1748275449539"/>
|
||||
</node>
|
||||
</node>
|
||||
<node TEXT="Paging algorhytmus" ID="6" CREATED="1748275449539" MODIFIED="1748275449539">
|
||||
<node TEXT="Es wird eine Reader benötigt, der sequentiell die Daten einließt." ID="10" CREATED="1748275449539" MODIFIED="1748275449539">
|
||||
<node TEXT="Reader ließt immer die aktuelle Page und im Hintergrund die potentiell nächste Page ein." ID="11" CREATED="1748275449539" MODIFIED="1748275449539">
|
||||
<node TEXT="Möglich wenig Latenz beim Zugriff." ID="16" CREATED="1748275449539" MODIFIED="1748275449539"/>
|
||||
<node TEXT="Es werden immer nur n Pages im Speicher gehalten. Wobei n >= 2 ist." ID="17" CREATED="1748275449539" MODIFIED="1748275449539"/>
|
||||
</node>
|
||||
<node TEXT="Dateien könnten extern aktualisiert worden sein." ID="19" CREATED="1748275449539" MODIFIED="1748275449539">
|
||||
<node TEXT="Die eingelesene Page muss dann verworfen werden." ID="20" CREATED="1748275449540" MODIFIED="1748275449540"/>
|
||||
</node>
|
||||
</node>
|
||||
<node TEXT="Jede Datei repräsentiert eine "Page"" ID="12" CREATED="1748275449540" MODIFIED="1748275449540">
|
||||
<node TEXT="Dateien werden also immer komplett eingelesen." ID="13" CREATED="1748275449540" MODIFIED="1748275449540"/>
|
||||
<node TEXT="Dateien müssen also nicht geblockt werden" ID="14" CREATED="1748275449540" MODIFIED="1748275449540">
|
||||
<node TEXT="Besser für Netzwerkverkehr und unterstützt Caching auf Betriebssystemebene." ID="15" CREATED="1748275449540" MODIFIED="1748275449540"/>
|
||||
</node>
|
||||
</node>
|
||||
</node>
|
||||
</node>
|
||||
<node TEXT="Es handelt sich um Sequenzen von gleichgroßen Datenpunkten, also um Records." ID="21" CREATED="1748275449540" MODIFIED="1748275449540">
|
||||
<node TEXT="Der Loader bekommt den Typ des Records als generischen Parameter T mit." ID="22" CREATED="1748275449540" MODIFIED="1748275449540"/>
|
||||
</node>
|
||||
</node>
|
||||
</node>
|
||||
</map>
|
||||
@@ -4,6 +4,7 @@ interface
|
||||
|
||||
uses
|
||||
System.SysUtils, System.Classes, System.Generics.Collections, System.StrUtils,
|
||||
Myc.Futures,
|
||||
Myc.TickLoader, Myc.OHLCCache; // OHLCCache for TOHLC, IGenericCandleBuilder, TF_ constants
|
||||
|
||||
type
|
||||
@@ -62,26 +63,25 @@ const
|
||||
{ TChartDataController }
|
||||
|
||||
constructor TChartDataController.Create(AMemoTarget: TStrings);
|
||||
var
|
||||
I: Integer; // Standard loop variable
|
||||
begin
|
||||
inherited Create;
|
||||
FMemoLog := AMemoTarget;
|
||||
FGenericBuilders := TDictionary<string, IGenericCandleBuilder>.Create;
|
||||
|
||||
InitializeBaseTimeframeSettings; // This populates FTimeframeSettingsArray
|
||||
|
||||
FOHLCCacheList := TList<TOHLC>.Create; // Create the TList object
|
||||
// Add nil placeholders for each timeframe setting defined in FTimeframeSettingsArray
|
||||
// This ensures FOHLCCacheList is parallel to FTimeframeSettingsArray from the start.
|
||||
for I := 0 to High(FTimeframeSettingsArray) do
|
||||
begin
|
||||
FOHLCCacheList.Add(nil); // Add a nil item to the list
|
||||
end;
|
||||
|
||||
SetLength(FTickDataArray, 0);
|
||||
if FMemoLog <> nil then
|
||||
FMemoLog.Add('TChartDataController instance created. Base timeframes initialized.');
|
||||
var
|
||||
I: Integer; // Standard loop variable
|
||||
begin
|
||||
inherited Create;
|
||||
FMemoLog := AMemoTarget;
|
||||
FGenericBuilders := TDictionary<string, IGenericCandleBuilder>.Create;
|
||||
|
||||
InitializeBaseTimeframeSettings; // This populates FTimeframeSettingsArray
|
||||
|
||||
FOHLCCacheList := TList<TOHLC>.Create; // Create the TList object
|
||||
// Add nil placeholders for each timeframe setting defined in FTimeframeSettingsArray
|
||||
// This ensures FOHLCCacheList is parallel to FTimeframeSettingsArray from the start.
|
||||
for I := 0 to High(FTimeframeSettingsArray) do
|
||||
begin
|
||||
FOHLCCacheList.Add(nil); // Add a nil item to the list
|
||||
end;
|
||||
|
||||
if FMemoLog <> nil then
|
||||
FMemoLog.Add('TChartDataController instance created. Base timeframes initialized.');
|
||||
end;
|
||||
|
||||
destructor TChartDataController.Destroy;
|
||||
@@ -211,16 +211,15 @@ procedure TChartDataController.LoadTickDataSeries(const AFileName: string);
|
||||
begin
|
||||
ClearAllDataAndCaches();
|
||||
try
|
||||
FTickDataArray := Myc.TickLoader.LoadTickDataSeries(AFileName);
|
||||
FTickDataArray := Myc.TickLoader.LoadTickDataSeries(AFileName).WaitFor;
|
||||
if FMemoLog <> nil then
|
||||
FMemoLog.Add(Format('DataController: Loaded %d tick records from series: %s',
|
||||
[Length(FTickDataArray), AFileName]));
|
||||
FMemoLog.Add(Format('DataController: Loading tick records from series: %s',
|
||||
[AFileName]));
|
||||
except
|
||||
on E: Exception do
|
||||
begin
|
||||
if FMemoLog <> nil then
|
||||
FMemoLog.Add(Format('DataController: Error loading tick data from %s - %s', [AFileName, E.Message]));
|
||||
SetLength(FTickDataArray, 0);
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
+386
-186
@@ -1,33 +1,42 @@
|
||||
unit Myc.TickLoader;
|
||||
|
||||
(*------------------------------------------------------------------------------
|
||||
Unit: Myc.TickLoader
|
||||
Author: Delphi Algo Trading (Assistant)
|
||||
Date: May 22, 2025
|
||||
(* ------------------------------------------------------------------------------
|
||||
Unit: Myc.TickLoader
|
||||
Author: Delphi Algo Trading (Assistant)
|
||||
Date: May 27, 2025
|
||||
Description:
|
||||
This unit provides functions to load tick data (TTickData) from
|
||||
binary files. The data originates from a C# application
|
||||
and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
|
||||
This unit provides functions to load tick data (TTickData) from
|
||||
binary files. The data originates from a C# application
|
||||
and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
|
||||
|
||||
The files now follow the naming convention: Symbol_Year.tab.
|
||||
(Prefix has been removed from the convention).
|
||||
Loading occurs in two phases:
|
||||
1. All .tab files are processed chronologically. Unique ticks are loaded,
|
||||
and the timestamp of the latest tick across all .tab files is determined
|
||||
(overallLastTabTickTime). Assumes timestamps are unique across all .tab files.
|
||||
2. All .tab-live files (for years corresponding to found .tab files) are
|
||||
processed. Ticks from a .tab-live file are only integrated if their
|
||||
timestamp is strictly later than overallLastTabTickTime. Assumes such
|
||||
ticks are globally unique.
|
||||
The files follow the naming convention:
|
||||
For .tab files: Symbol_Year_MM.tab OR Symbol_Year_MM.tab_zip
|
||||
For .tab-live files: Symbol_Year_MM.tab-live (NEVER compressed)
|
||||
(MM represents the two-digit month, e.g., 01 for January, 12 for December)
|
||||
|
||||
If a .tab-live file results in no new data being added under these rules,
|
||||
it is deleted after processing.
|
||||
If both compressed (.tab_zip) and uncompressed (.tab) versions of a .tab
|
||||
file exist, the compressed version is preferred. Files ending with _zip
|
||||
are expected to be ZIP archives containing the actual data file
|
||||
(e.g., Symbol_Year_MM.tab).
|
||||
|
||||
The loading function automatically detects subsequent yearly .tab files.
|
||||
Loading occurs in two phases:
|
||||
1. All .tab files (regular or _zip) are processed chronologically by
|
||||
month and year. Unique ticks are loaded, and the timestamp of the
|
||||
latest tick across all .tab files is determined (overallLastTabTickTime).
|
||||
Assumes timestamps are unique across all .tab files.
|
||||
2. All .tab-live files (which are never compressed), for months/years
|
||||
corresponding to found .tab files, are processed. Ticks from a
|
||||
.tab-live file are only integrated if their timestamp is strictly later
|
||||
than overallLastTabTickTime. Assumes such ticks are globally unique.
|
||||
|
||||
If a .tab-live file results in no new data being added under these rules,
|
||||
it is deleted after processing.
|
||||
|
||||
The loading function automatically detects subsequent monthly/yearly files.
|
||||
|
||||
Main Function:
|
||||
LoadTickDataSeries - Loads a series of tick data files.
|
||||
------------------------------------------------------------------------------*)
|
||||
LoadTickDataSeries - Loads a series of tick data files.
|
||||
------------------------------------------------------------------------------ *)
|
||||
|
||||
interface
|
||||
|
||||
@@ -35,7 +44,9 @@ uses
|
||||
System.SysUtils,
|
||||
System.Classes,
|
||||
System.Generics.Collections,
|
||||
System.IOUtils;
|
||||
System.IOUtils,
|
||||
Myc.Futures, // Added for TFuture in ReadFileTickData
|
||||
Myc.Signals; // Added for IMycState (dependency of TFuture)
|
||||
|
||||
type
|
||||
TTickData = packed record
|
||||
@@ -43,211 +54,400 @@ type
|
||||
Ask: Single;
|
||||
Bid: Single;
|
||||
end;
|
||||
|
||||
PTTickData = ^TTickData;
|
||||
|
||||
// TryParseFileName now expects filenames like "Symbol_Year" (or "Symbol_Part2_Year")
|
||||
// PrefixValue will be returned as an empty string.
|
||||
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue: Integer): Boolean;
|
||||
function LoadTickDataSeries(const FileName: string): TArray<TTickData>;
|
||||
// TryParseFileName parses filenames like "Symbol_Year_MM.Extension" or "Symbol_Year_MM.Extension_zip"
|
||||
// to extract Path, Symbol, Year, and Month.
|
||||
function TryParseFileName( const FileName: string;
|
||||
out PathValue, SymbolValue: string;
|
||||
out YearValue, MonthValue: Integer ): Boolean;
|
||||
|
||||
function LoadTickDataSeries(const FileName: string): TFuture<TArray<TTickData>>;
|
||||
|
||||
// Asynchronously reads tick data from a single specified file.
|
||||
// Returns a TFuture that will provide an array of TTickData.
|
||||
function ReadFileTickData( var LoadGate: TLatch; const FileName: string ): TFuture<TArray<TTickData>>;
|
||||
|
||||
implementation
|
||||
|
||||
function TryParseFileName(const FileName: string; out PathValue, SymbolValue: string; out YearValue: Integer): Boolean;
|
||||
uses
|
||||
System.Zip,
|
||||
// Myc.Futures, Myc.Signals already in interface uses for TFuture type visibility
|
||||
Myc.TaskManager; // Added for the global TaskManager used by TFuture.Construct
|
||||
|
||||
function TryParseFileName( const FileName: string;
|
||||
out PathValue, SymbolValue: string;
|
||||
out YearValue, MonthValue: Integer ): Boolean;
|
||||
var
|
||||
fileNameOnly: string;
|
||||
fileNameNoPath: string;
|
||||
nameForParsing: string;
|
||||
parts: TArray<string>;
|
||||
begin
|
||||
Result := False;
|
||||
PathValue := TPath.GetDirectoryName(FileName);
|
||||
fileNameOnly := TPath.GetFileNameWithoutExtension(FileName); // Handles any extension for parsing basename
|
||||
Result := False; // Default to failure
|
||||
PathValue := ''; // Initialize all out parameters
|
||||
SymbolValue := '';
|
||||
YearValue := 0;
|
||||
MonthValue := 0;
|
||||
|
||||
parts := fileNameOnly.Split(['_']);
|
||||
PathValue := TPath.GetDirectoryName( FileName );
|
||||
fileNameNoPath := TPath.GetFileName( FileName );
|
||||
|
||||
// Expected format is Symbol_Year or SymbolPart1_SymbolPart2_Year, so at least 2 parts.
|
||||
if Length(parts) < 2 then
|
||||
nameForParsing := fileNameNoPath;
|
||||
if nameForParsing.EndsWith( '_zip', True ) then
|
||||
begin
|
||||
SymbolValue := ''; YearValue := 0; // Clear out params on minimum parts failure
|
||||
Exit;
|
||||
nameForParsing := nameForParsing.Substring( 0, nameForParsing.Length - 4 );
|
||||
end;
|
||||
|
||||
// Year is the last part.
|
||||
if not TryStrToInt(parts[High(parts)], YearValue) then
|
||||
nameForParsing := TPath.GetFileNameWithoutExtension( nameForParsing ); // e.g., "EURUSD_2023_01"
|
||||
|
||||
parts := nameForParsing.Split( ['_'] );
|
||||
|
||||
if Length( parts ) < 3 then // Need at least Symbol_Year_Month
|
||||
begin
|
||||
SymbolValue := ''; // Clear SymbolValue as well on year parse failure
|
||||
Exit;
|
||||
exit;
|
||||
end;
|
||||
|
||||
// Symbol consists of all parts before the year part.
|
||||
// Copy all parts except the last one (which is the Year).
|
||||
// The second parameter of Copy (for TArray<string>) is the starting index,
|
||||
// the third parameter is the count of elements to copy.
|
||||
// We want to copy elements from index 0 up to (but not including) the last element.
|
||||
// So, the count is High(parts) (which is Length(parts) - 1).
|
||||
if High(parts) > 0 then // Ensure there's at least one part for the symbol
|
||||
SymbolValue := string.Join('_', Copy(parts, 0, High(parts)))
|
||||
else if Length(parts) = 1 then // Edge case: File is "Year.tab" - this is invalid by new rule "Symbol_Year"
|
||||
begin SymbolValue := ''; Exit; end // Or handle as Symbol = parts[0] if "Year" itself is a symbol
|
||||
else // Should not happen if Length(parts) < 2 already exited
|
||||
SymbolValue := '';
|
||||
// Month is the last part.
|
||||
if not TryStrToInt( parts[High( parts )], MonthValue ) then
|
||||
begin
|
||||
exit;
|
||||
end;
|
||||
if ( MonthValue < 1 ) or ( MonthValue > 12 ) then // Validate month range
|
||||
begin
|
||||
MonthValue := 0; // Set to invalid if out of range
|
||||
exit;
|
||||
end;
|
||||
|
||||
// Year is the second to last part.
|
||||
if not TryStrToInt( parts[High( parts ) - 1], YearValue ) then
|
||||
begin
|
||||
MonthValue := 0; // Also invalidate month if year parsing fails
|
||||
exit;
|
||||
end;
|
||||
|
||||
// Symbol consists of all parts before Year and Month. Count = Length(parts) - 2
|
||||
SymbolValue := string.Join( '_', Copy( parts, 0, Length( parts ) - 2 ) );
|
||||
|
||||
if SymbolValue = '' then // Ensure symbol is not empty
|
||||
begin
|
||||
exit;
|
||||
end;
|
||||
|
||||
Result := True;
|
||||
end;
|
||||
|
||||
function LoadTickDataSeries(const FileName: string): TArray<TTickData>;
|
||||
var
|
||||
tickList: TList<TTickData>;
|
||||
initialYear, currentTabYear, liveProcessingYear: Integer;
|
||||
pathName, symbolName: string; // prefixNameOut will be an empty out param
|
||||
tabFileName, liveFileName: string;
|
||||
overallLastTabTickTime: Double;
|
||||
maxTabYearFound: Integer;
|
||||
|
||||
procedure ReadFileContentAndGetMaxTime(const AFileName: string;
|
||||
out Records: TArray<TTickData>;
|
||||
out ActualMaxTickTimeInFile: Double);
|
||||
var
|
||||
fs: TFileStream;
|
||||
fileSize: Int64;
|
||||
recordCount: Integer;
|
||||
bytesRead: Integer;
|
||||
iterTickData: TTickData;
|
||||
currentFileMaxOA: Double;
|
||||
begin
|
||||
SetLength(Records, 0);
|
||||
ActualMaxTickTimeInFile := 0.0;
|
||||
currentFileMaxOA := 0.0;
|
||||
|
||||
if not TFile.Exists(AFileName) then // Added check for file existence
|
||||
begin
|
||||
// Silently exit or log if preferred, but don't try to open non-existent file.
|
||||
Exit;
|
||||
end;
|
||||
|
||||
fs := TFileStream.Create(AFileName, fmOpenRead or fmShareDenyWrite);
|
||||
try
|
||||
fileSize := fs.Size;
|
||||
if fileSize = 0 then Exit;
|
||||
|
||||
if (fileSize mod SizeOf(TTickData)) <> 0 then
|
||||
raise EReadError.CreateFmt('File %s: Corrupted. Size %d is not a multiple of record size %d.',
|
||||
[AFileName, fileSize, SizeOf(TTickData)]);
|
||||
|
||||
recordCount := fileSize div SizeOf(TTickData);
|
||||
if recordCount > 0 then
|
||||
begin
|
||||
SetLength(Records, recordCount);
|
||||
bytesRead := fs.Read(Records[0], fileSize);
|
||||
if bytesRead <> fileSize then
|
||||
raise EReadError.CreateFmt('File %s: Read error. Expected to read %d bytes, but actually read %d bytes.',
|
||||
[AFileName, fileSize, bytesRead]);
|
||||
|
||||
for iterTickData in Records do
|
||||
begin
|
||||
if iterTickData.OADateTime > currentFileMaxOA then
|
||||
currentFileMaxOA := iterTickData.OADateTime;
|
||||
end;
|
||||
ActualMaxTickTimeInFile := currentFileMaxOA;
|
||||
end;
|
||||
finally
|
||||
fs.Free;
|
||||
end;
|
||||
function LoadTickDataSeries(const FileName: string): TFuture<TArray<TTickData>>;
|
||||
// Local type declaration for storing file information for phase 1 processing
|
||||
type
|
||||
TPhase1FileEntry = record
|
||||
Path: string;
|
||||
Year: Integer;
|
||||
Month: Integer;
|
||||
end;
|
||||
var
|
||||
initialYear, initialMonth: Integer;
|
||||
currentTabYear, currentTabMonth: Integer;
|
||||
pathName, symbolName: string;
|
||||
tabFileEntryRecord: TPhase1FileEntry; // Variable to construct record instances
|
||||
loadedState: TState;
|
||||
loadedFiles: TArray<TFuture<TArray<TTickData>>>;
|
||||
liveData: TFuture<TArray<TTickData>>;
|
||||
|
||||
begin // Start of LoadTickDataSeries
|
||||
tickList := TList<TTickData>.Create;
|
||||
overallLastTabTickTime := 0.0;
|
||||
maxTabYearFound := -1;
|
||||
if not TryParseFileName( FileName, pathName, symbolName, initialYear, initialMonth ) then
|
||||
begin
|
||||
raise EArgumentException.CreateFmt(
|
||||
'Invalid initial file name format: %s. Expected Symbol_Year_MM.Extension or Symbol_Year_MM.Extension_zip', [FileName] );
|
||||
end;
|
||||
|
||||
currentTabYear := initialYear;
|
||||
currentTabMonth := initialMonth;
|
||||
var tabBaseNameFormat: string;
|
||||
var compressedTabPath, uncompressedTabPath: string;
|
||||
var actualTabFileToLoad: string;
|
||||
var maxTabYearFound := -1;
|
||||
var maxTabMonthFound := -1;
|
||||
|
||||
var filesToLoadPhase1 := TList<TPhase1FileEntry>.Create; // Create list with the named record type
|
||||
try
|
||||
// Initial FileName must be the path to the first .tab file (e.g., Symbol_Year.tab)
|
||||
// prefixNameOut will be an out parameter from TryParseFileName but will be empty.
|
||||
if not TryParseFileName(FileName, pathName, symbolName, initialYear) then
|
||||
while True do
|
||||
begin
|
||||
raise EArgumentException.CreateFmt('Invalid initial file name format: %s. Expected Symbol_Year.tab (or Symbol_Part2_Year.tab)', [FileName]);
|
||||
end;
|
||||
tabBaseNameFormat := Format( '%s_%.4d_%.2d.tab', [symbolName, currentTabYear, currentTabMonth] );
|
||||
compressedTabPath := TPath.Combine( pathName, tabBaseNameFormat + '_zip' );
|
||||
uncompressedTabPath := TPath.Combine( pathName, tabBaseNameFormat );
|
||||
|
||||
// --- PHASE 1: Load all .tab files and determine overallLastTabTickTime ---
|
||||
currentTabYear := initialYear;
|
||||
// Construct the first .tab file name using the parsed components (symbol, year).
|
||||
// Prefix is no longer used in the format string.
|
||||
tabFileName := TPath.Combine(pathName, Format('%s_%d.tab', [symbolName, currentTabYear]));
|
||||
|
||||
var tabFileRecords: TArray<TTickData>;
|
||||
var maxTimeInThisTabFile: Double;
|
||||
|
||||
while TFile.Exists(tabFileName) do
|
||||
begin
|
||||
maxTabYearFound := currentTabYear;
|
||||
ReadFileContentAndGetMaxTime(tabFileName, tabFileRecords, maxTimeInThisTabFile);
|
||||
|
||||
if Length(tabFileRecords) > 0 then
|
||||
actualTabFileToLoad := '';
|
||||
if TFile.Exists( compressedTabPath ) then
|
||||
begin
|
||||
tickList.AddRange(tabFileRecords);
|
||||
actualTabFileToLoad := compressedTabPath;
|
||||
end
|
||||
else if TFile.Exists( uncompressedTabPath ) then
|
||||
begin
|
||||
actualTabFileToLoad := uncompressedTabPath;
|
||||
end;
|
||||
|
||||
if maxTimeInThisTabFile > overallLastTabTickTime then
|
||||
overallLastTabTickTime := maxTimeInThisTabFile;
|
||||
if actualTabFileToLoad = '' then
|
||||
break; // No more sequential .tab files
|
||||
|
||||
Inc(currentTabYear);
|
||||
// Construct next tab filename without prefix
|
||||
tabFileName := TPath.Combine(pathName, Format('%s_%d.tab', [symbolName, currentTabYear]));
|
||||
// Construct the record and add it to the list
|
||||
tabFileEntryRecord.Path := actualTabFileToLoad;
|
||||
tabFileEntryRecord.Year := currentTabYear;
|
||||
tabFileEntryRecord.Month := currentTabMonth;
|
||||
filesToLoadPhase1.Add( tabFileEntryRecord );
|
||||
|
||||
// Advance to the next month
|
||||
Inc( currentTabMonth );
|
||||
if currentTabMonth > 12 then
|
||||
begin
|
||||
currentTabMonth := 1;
|
||||
Inc( currentTabYear );
|
||||
end;
|
||||
end;
|
||||
|
||||
// --- PHASE 2: Load .tab-live files ---
|
||||
if maxTabYearFound >= initialYear then
|
||||
begin
|
||||
for liveProcessingYear := initialYear to maxTabYearFound do
|
||||
// Now load and process identified .tab files
|
||||
var loadedFileList := TList<TFuture<TArray<TTickData>>>.Create;
|
||||
var loadStates := TList<TState>.Create;
|
||||
try
|
||||
var LoadGate: TLatch;
|
||||
|
||||
for var tabFileEntry: TPhase1FileEntry in filesToLoadPhase1 do // Iterate with the correct type
|
||||
begin
|
||||
// Construct .tab-live filename without prefix
|
||||
liveFileName := TPath.Combine(pathName, Format('%s_%d.tab-live', [symbolName, liveProcessingYear]));
|
||||
var data := ReadFileTickData( LoadGate, tabFileEntry.Path );
|
||||
|
||||
if TFile.Exists(liveFileName) then
|
||||
loadedFileList.Add( TFuture<TArray<TTickData>>.Create( data ) );
|
||||
loadStates.Add( data.Done );
|
||||
|
||||
if tabFileEntry.Year > maxTabYearFound then
|
||||
begin
|
||||
var liveFileRecords: TArray<TTickData>;
|
||||
var maxTimeInThisLiveFile: Double;
|
||||
var ticksToActuallyAddFromLive: TList<TTickData>;
|
||||
var liveDataAddedThisFile: Boolean;
|
||||
var iterLiveTickData: TTickData;
|
||||
|
||||
ReadFileContentAndGetMaxTime(liveFileName, liveFileRecords, maxTimeInThisLiveFile);
|
||||
liveDataAddedThisFile := False;
|
||||
|
||||
if Length(liveFileRecords) > 0 then
|
||||
begin
|
||||
ticksToActuallyAddFromLive := TList<TTickData>.Create;
|
||||
try
|
||||
for iterLiveTickData in liveFileRecords do
|
||||
begin
|
||||
if iterLiveTickData.OADateTime > overallLastTabTickTime then
|
||||
begin
|
||||
ticksToActuallyAddFromLive.Add(iterLiveTickData);
|
||||
liveDataAddedThisFile := True;
|
||||
end;
|
||||
end;
|
||||
|
||||
if ticksToActuallyAddFromLive.Count > 0 then
|
||||
tickList.AddRange(ticksToActuallyAddFromLive);
|
||||
finally
|
||||
ticksToActuallyAddFromLive.Free;
|
||||
end;
|
||||
end;
|
||||
|
||||
if not liveDataAddedThisFile then
|
||||
begin
|
||||
try
|
||||
TFile.Delete(liveFileName);
|
||||
except
|
||||
on E: Exception do {/* Optional: Log error, e.g., using a global logger or passed logger */}
|
||||
end;
|
||||
end;
|
||||
maxTabYearFound := tabFileEntry.Year;
|
||||
maxTabMonthFound := tabFileEntry.Month;
|
||||
end
|
||||
else if ( tabFileEntry.Year = maxTabYearFound ) and ( tabFileEntry.Month > maxTabMonthFound ) then
|
||||
begin
|
||||
maxTabMonthFound := tabFileEntry.Month;
|
||||
end;
|
||||
end;
|
||||
|
||||
var liveFileBaseName := Format( '%s_%d_%02d.tab-live', [symbolName, maxTabYearFound, maxTabMonthFound] );
|
||||
liveData := ReadFileTickData( LoadGate, TPath.Combine( pathName, liveFileBaseName ) );
|
||||
loadStates.Add( liveData.Done );
|
||||
loadedFileList.Add( liveData );
|
||||
|
||||
loadedState := TState.All( loadStates.ToArray );
|
||||
loadedFiles := loadedFileList.ToArray;
|
||||
finally
|
||||
loadStates.Free;
|
||||
loadedFileList.Free;
|
||||
end;
|
||||
finally
|
||||
filesToLoadPhase1.Free;
|
||||
end;
|
||||
|
||||
Result := TFuture<TArray<TTickData>>.Construct(
|
||||
loadedState,
|
||||
function: TArray<TTickData>
|
||||
begin
|
||||
var tickList := TList<TTickData>.Create;
|
||||
try
|
||||
var totalTicks := Length(liveData.Result);
|
||||
var overallLastTabTickTime := 0.0;
|
||||
|
||||
for var P in loadedFiles do
|
||||
inc( totalTicks, Length(P.Result) );
|
||||
|
||||
tickList.Capacity := totalTicks;
|
||||
|
||||
for var P in loadedFiles do
|
||||
begin
|
||||
if Length(P.Result) > 0 then
|
||||
begin
|
||||
inc( totalTicks, Length(P.Result) );
|
||||
|
||||
for var iterTickData in P.Result do
|
||||
if iterTickData.OADateTime > overallLastTabTickTime then
|
||||
begin
|
||||
tickList.Add( iterTickData );
|
||||
overallLastTabTickTime := iterTickData.OADateTime;
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
Result := tickList.ToArray;
|
||||
finally
|
||||
tickList.Free;
|
||||
end;
|
||||
end );
|
||||
end;
|
||||
|
||||
function LoadSingleFileTickDataFromStream(const InputStream: TStream): TArray<TTickData>;
|
||||
var
|
||||
fileSize: Int64;
|
||||
recordCount: Integer;
|
||||
bytesRead: Integer;
|
||||
begin
|
||||
// Initialize Result record
|
||||
SetLength(Result, 0);
|
||||
|
||||
InputStream.Position := 0;
|
||||
|
||||
fileSize := InputStream.Size;
|
||||
if fileSize = 0 then
|
||||
exit;
|
||||
|
||||
if (fileSize mod SizeOf(TTickData)) <> 0 then
|
||||
raise EReadError.CreateFmt('File corrupted. Size %d is not a multiple of record size %d.',
|
||||
[fileSize, SizeOf(TTickData)]);
|
||||
|
||||
recordCount := fileSize div SizeOf(TTickData);
|
||||
if recordCount > 0 then
|
||||
begin
|
||||
SetLength(Result, recordCount);
|
||||
bytesRead := InputStream.Read(Result[0], fileSize);
|
||||
if bytesRead <> fileSize then
|
||||
raise EReadError.CreateFmt('Read error. Expected to read %d bytes, but actually read %d bytes.',
|
||||
[fileSize, bytesRead]);
|
||||
end;
|
||||
end;
|
||||
|
||||
function LoadUncompressedSingleFileTickData(const FullFileName: string): TArray<TTickData>;
|
||||
var
|
||||
inputStream: TStream;
|
||||
begin
|
||||
// Initialize Result record
|
||||
SetLength(Result, 0);
|
||||
|
||||
if not TFile.Exists(FullFileName) then
|
||||
begin
|
||||
exit;
|
||||
end;
|
||||
|
||||
inputStream := TFileStream.Create(FullFileName, fmOpenRead or fmShareDenyWrite);
|
||||
try
|
||||
try
|
||||
Result := LoadSingleFileTickDataFromStream( inputStream );
|
||||
|
||||
except
|
||||
on E: EReadError do
|
||||
begin
|
||||
raise EReadError.CreateFmt('File %s: %s', [FullFileName, E.Message]);
|
||||
end;
|
||||
end;
|
||||
finally
|
||||
inputStream.Free;
|
||||
end;
|
||||
end;
|
||||
|
||||
function UncompressTickDataFromStream(const InputStream: TStream): TArray<TTickData>;
|
||||
var
|
||||
decompressionStream: TStream; // Holds uncompressed data of a single zip entry
|
||||
localHeader: TZipHeader;
|
||||
entryIndex: Integer;
|
||||
i: Integer;
|
||||
zipFileInstance: TZipFile;
|
||||
begin
|
||||
// Initialize Result record
|
||||
SetLength(Result, 0);
|
||||
|
||||
decompressionStream := nil;
|
||||
zipFileInstance := nil;
|
||||
|
||||
try
|
||||
InputStream.Position := 0; // Reset stream position to the beginning for TZipFile
|
||||
|
||||
// Open TZipFile from this memory stream
|
||||
zipFileInstance := TZipFile.Create;
|
||||
zipFileInstance.Open(InputStream, TZipMode.zmRead); // Open the zip from the memory stream
|
||||
|
||||
if zipFileInstance.FileCount = 0 then
|
||||
begin
|
||||
// If TZipFile opens an empty or invalid stream successfully but finds no files
|
||||
exit; // FileSuccessfullyLoaded remains False
|
||||
end;
|
||||
|
||||
Result := tickList.ToArray;
|
||||
entryIndex := -1;
|
||||
for i := 0 to zipFileInstance.FileCount - 1 do
|
||||
begin
|
||||
if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
|
||||
begin
|
||||
entryIndex := i;
|
||||
break;
|
||||
end;
|
||||
end;
|
||||
|
||||
if entryIndex = -1 then // If specific name not found, default to the first entry
|
||||
begin
|
||||
if zipFileInstance.FileCount > 0 then
|
||||
entryIndex := 0
|
||||
else
|
||||
exit; // Should not happen if FileCount > 0 check passed
|
||||
end;
|
||||
|
||||
// Decompress the specific entry from the in-memory zip file.
|
||||
// TZipFile.Read creates and populates decompressionStream with uncompressed data.
|
||||
zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
|
||||
|
||||
if not Assigned(decompressionStream) then
|
||||
exit; // Exit if no valid stream could be prepared
|
||||
|
||||
Result := LoadSingleFileTickDataFromStream( decompressionStream );
|
||||
finally
|
||||
tickList.Free;
|
||||
// Ensure all potentially created objects are freed
|
||||
if Assigned(decompressionStream) then // Holds uncompressed data from a zip entry
|
||||
decompressionStream.Free;
|
||||
if Assigned(zipFileInstance) then
|
||||
zipFileInstance.Free;
|
||||
end;
|
||||
end;
|
||||
|
||||
function ReadFileTickData( var LoadGate: TLatch; const FileName: string ): TFuture<TArray<TTickData>>;
|
||||
var
|
||||
parsedPath, parsedSymbol: string;
|
||||
parsedYear, parsedMonth: Integer;
|
||||
begin
|
||||
Result := TFuture<TArray<TTickData>>.Null;
|
||||
|
||||
// Attempt to parse year and month from the filename
|
||||
if not TryParseFileName( FileName, parsedPath, parsedSymbol, parsedYear, parsedMonth ) then
|
||||
begin
|
||||
// If filename parsing fails, return a future that resolves to an empty array.
|
||||
exit;
|
||||
end;
|
||||
|
||||
if FileName.EndsWith('_zip', True) then
|
||||
begin
|
||||
var zipFileBytes := TFile.ReadAllBytes(FileName); // Read all bytes of the .zip file
|
||||
if Length(zipFileBytes) = 0 then // Handle empty zip file case
|
||||
exit;
|
||||
|
||||
var capFileName := FileName;
|
||||
var blob := TFuture<TBytes>.Construct(
|
||||
TLatch.Enqueue( LoadGate ),
|
||||
function: TBytes
|
||||
begin
|
||||
Result := TFile.ReadAllBytes(capFileName); // Read all bytes of the .zip file
|
||||
end );
|
||||
|
||||
Result := blob.Chain<TArray<TTickData>>(
|
||||
function( bytes: TBytes ): TArray<TTickData>
|
||||
begin
|
||||
var zipMemoryStream := TBytesStream.Create( zipFileBytes ); // Create a memory stream to hold the zip file content
|
||||
try
|
||||
zipMemoryStream.Position := 0;
|
||||
Result := UncompressTickDataFromStream( zipMemoryStream );
|
||||
finally
|
||||
zipMemoryStream.Free;
|
||||
end;
|
||||
end );
|
||||
end
|
||||
else
|
||||
begin
|
||||
var capFileName := FileName;
|
||||
Result := TFuture<TArray<TTickData>>.Construct(
|
||||
function: TArray<TTickData>
|
||||
begin
|
||||
Result := LoadUncompressedSingleFileTickData( capFileName );
|
||||
end );
|
||||
end;
|
||||
end;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user