Refactoring to new TDataSeries
This commit is contained in:
Michael Schimmel
@@ -16,14 +16,14 @@ object ChartForm: TChartForm
|
||||
900
|
||||
835)
|
||||
TextHeight = 15
|
||||
object FileSelectButton: TSpeedButton
|
||||
Left = 647
|
||||
Top = 17
|
||||
object PathSelectButton: TSpeedButton
|
||||
Left = 423
|
||||
Top = 18
|
||||
Width = 23
|
||||
Height = 22
|
||||
Height = 23
|
||||
Anchors = [akTop, akRight]
|
||||
Caption = '...'
|
||||
OnClick = FileSelectButtonClick
|
||||
OnClick = PathSelectButtonClick
|
||||
end
|
||||
object MouseHoverDataLabel: TLabel
|
||||
Left = 32
|
||||
@@ -34,14 +34,14 @@ object ChartForm: TChartForm
|
||||
AutoSize = False
|
||||
ExplicitTop = 359
|
||||
end
|
||||
object FileNameEdit: TEdit
|
||||
object DataPathEdit: TEdit
|
||||
Left = 32
|
||||
Top = 17
|
||||
Width = 609
|
||||
Width = 385
|
||||
Height = 23
|
||||
Anchors = [akLeft, akTop, akRight]
|
||||
TabOrder = 0
|
||||
Text = '\\COFFEE\TickData\Pepperstone\EURUSD_2017_02.tab_zip'
|
||||
Text = '\\COFFEE\TickData\Pepperstone'
|
||||
end
|
||||
object LoadButton: TButton
|
||||
Left = 686
|
||||
@@ -116,4 +116,14 @@ object ChartForm: TChartForm
|
||||
ExplicitHeight = 97
|
||||
end
|
||||
end
|
||||
object SymbolComboBox: TComboBox
|
||||
Left = 452
|
||||
Top = 18
|
||||
Width = 228
|
||||
Height = 23
|
||||
Anchors = [akTop, akRight]
|
||||
TabOrder = 7
|
||||
Text = 'GER40'
|
||||
OnChange = SymbolComboBoxChange
|
||||
end
|
||||
end
|
||||
|
||||
+774
-488
File diff suppressed because it is too large
Load Diff
@@ -5,7 +5,7 @@ interface
|
||||
uses
|
||||
System.SysUtils, System.Classes, System.Generics.Collections, System.StrUtils,
|
||||
Myc.Futures,
|
||||
Myc.TickLoader, Myc.OHLCCache; // OHLCCache for TOHLC, IGenericCandleBuilder, TF_ constants
|
||||
Myc.Trade.DataSeries, Myc.OHLCCache;
|
||||
|
||||
type
|
||||
TTimeframeSetting = record
|
||||
@@ -16,7 +16,7 @@ type
|
||||
|
||||
TChartDataController = class
|
||||
private
|
||||
FTickDataArray: TArray<TTickData>;
|
||||
FTickDataArray: TArray<TAskBid.TTick>;
|
||||
FOHLCCacheList: TList<TOHLC>;
|
||||
FTimeframeSettingsArray: TArray<TTimeframeSetting>; // Now includes registered generic builders
|
||||
FMemoLog: TStrings;
|
||||
@@ -39,9 +39,8 @@ type
|
||||
|
||||
procedure ClearAllDataAndCaches;
|
||||
function GetTickDataCount: Integer;
|
||||
function GetTickDataArrayPtr: PTTickData;
|
||||
function GetTickDataLength: Integer;
|
||||
function GetTickDataItem(AIndex: Integer): TTickData;
|
||||
function GetTickDataItem(AIndex: Integer): TAskBid.TTick;
|
||||
function HasData: Boolean;
|
||||
|
||||
// ABuilderCaption is the unique key used for FTimeframeSettingsArray.Caption and FGenericBuilders key.
|
||||
@@ -55,7 +54,7 @@ type
|
||||
implementation
|
||||
|
||||
uses
|
||||
System.DateUtils; // For FormatDateTime in logging
|
||||
System.DateUtils;
|
||||
|
||||
const
|
||||
dcSecsPerDayConst = 24 * 60 * 60;
|
||||
@@ -211,7 +210,7 @@ procedure TChartDataController.LoadTickDataSeries(const AFileName: string);
|
||||
begin
|
||||
ClearAllDataAndCaches();
|
||||
try
|
||||
FTickDataArray := Myc.TickLoader.LoadTickDataSeries(AFileName).WaitFor;
|
||||
FTickDataArray := TAskBid.LoadDataSeries(AFileName).WaitFor();
|
||||
if FMemoLog <> nil then
|
||||
FMemoLog.Add(Format('DataController: Loading tick records from series: %s',
|
||||
[AFileName]));
|
||||
@@ -401,20 +400,12 @@ begin
|
||||
Result := Length(FTickDataArray);
|
||||
end;
|
||||
|
||||
function TChartDataController.GetTickDataArrayPtr: PTTickData;
|
||||
begin
|
||||
if Length(FTickDataArray) > 0 then
|
||||
Result := @FTickDataArray[0]
|
||||
else
|
||||
Result := nil;
|
||||
end;
|
||||
|
||||
function TChartDataController.GetTickDataLength: Integer;
|
||||
begin
|
||||
Result := Length(FTickDataArray);
|
||||
end;
|
||||
|
||||
function TChartDataController.GetTickDataItem(AIndex: Integer): TTickData;
|
||||
function TChartDataController.GetTickDataItem(AIndex: Integer): TAskBid.TTick;
|
||||
begin
|
||||
if (AIndex >= 0) and (AIndex < Length(FTickDataArray)) then
|
||||
Result := FTickDataArray[AIndex]
|
||||
@@ -422,7 +413,7 @@ begin
|
||||
begin
|
||||
if FMemoLog <> nil then
|
||||
FMemoLog.Add(Format('DataController: Attempted to access invalid tick data index %d. Count is %d.', [AIndex, Length(FTickDataArray)]));
|
||||
Result := Default(TTickData);
|
||||
Result := Default(TAskBid.TTick);
|
||||
end;
|
||||
end;
|
||||
|
||||
|
||||
+44
-45
@@ -35,7 +35,7 @@ interface
|
||||
uses
|
||||
System.Classes, System.SysUtils, System.Math,
|
||||
System.Generics.Collections, System.DateUtils,
|
||||
Myc.TickLoader;
|
||||
Myc.Trade.DataSeries;
|
||||
|
||||
const
|
||||
TF_AUTO_AGGREGATION = 0;
|
||||
@@ -56,9 +56,8 @@ type
|
||||
|
||||
// Interface for generic candle building logic
|
||||
IGenericCandleBuilder = interface
|
||||
['{B4F6A9C1-0A7E-42D3-847B-170705A8CC8D}'] // Added GUID
|
||||
function Init(const ADataSet: TArray<TTickData>): Boolean; // Changed DataSet to ADataSet for clarity
|
||||
function IsNewBar(AIndex: Integer; const ATick: TTickData): Boolean; // Changed Idx to AIndex, Tick to ATick
|
||||
function Init(const ADataSet: TArray<TAskBid.TTick>): Boolean; // Changed DataSet to ADataSet for clarity
|
||||
function IsNewBar(AIndex: Integer; const ATick: TAskBid.TTick): Boolean; // Changed Idx to AIndex, Tick to ATick
|
||||
function GetCaption: string; // Method for the Caption property
|
||||
property Caption: string read GetCaption;
|
||||
end;
|
||||
@@ -81,7 +80,7 @@ type
|
||||
public
|
||||
constructor Create;
|
||||
procedure ClearCache;
|
||||
procedure Build(const ATickData: TArray<TTickData>;
|
||||
procedure Build(const ATickData: TArray<TAskBid.TTick>;
|
||||
ASelectedTimeframeSeconds: Int64;
|
||||
AAutoAggCandleWidth: Integer;
|
||||
AAutoAggCandleSpacing: Integer;
|
||||
@@ -189,7 +188,7 @@ begin
|
||||
Result := Length(FCachedCandles);
|
||||
end;
|
||||
|
||||
procedure TOHLC.Build(const ATickData: TArray<TTickData>;
|
||||
procedure TOHLC.Build(const ATickData: TArray<TAskBid.TTick>;
|
||||
ASelectedTimeframeSeconds: Int64;
|
||||
AAutoAggCandleWidth: Integer; AAutoAggCandleSpacing: Integer; AAutoAggPaintBoxClientWidth: Integer;
|
||||
AMemoForLogging: TStrings; AGenericCandleBuilder: IGenericCandleBuilder = nil); // Parameter changed
|
||||
@@ -201,7 +200,7 @@ var
|
||||
CandleList: TList<TCachedCandle>;
|
||||
TempCandle: TCachedCandle;
|
||||
LogTimePerCandleStr: string;
|
||||
CurrentTick: TTickData;
|
||||
CurrentTick: TAskBid.TTick;
|
||||
begin
|
||||
if AMemoForLogging <> nil then
|
||||
AMemoForLogging.Add(FormatDateTime('yyyy-mm-dd hh:nn:ss', Now) + Format(' - TOHLC.Build: TF Secs: %d...',
|
||||
@@ -217,14 +216,14 @@ begin
|
||||
Exit;
|
||||
end;
|
||||
|
||||
FGlobalMinY := ATickData[0].Ask;
|
||||
FGlobalMaxY := ATickData[0].Ask;
|
||||
FGlobalMinY := ATickData[0].Data.Ask; // Changed
|
||||
FGlobalMaxY := ATickData[0].Data.Ask; // Changed
|
||||
for I := 1 to High(ATickData) do
|
||||
begin
|
||||
if ATickData[I].Ask < FGlobalMinY then
|
||||
FGlobalMinY := ATickData[I].Ask;
|
||||
if ATickData[I].Ask > FGlobalMaxY then
|
||||
FGlobalMaxY := ATickData[I].Ask;
|
||||
if ATickData[I].Data.Ask < FGlobalMinY then // Changed
|
||||
FGlobalMinY := ATickData[I].Data.Ask; // Changed
|
||||
if ATickData[I].Data.Ask > FGlobalMaxY then // Changed
|
||||
FGlobalMaxY := ATickData[I].Data.Ask; // Changed
|
||||
end;
|
||||
if FGlobalMaxY = FGlobalMinY then
|
||||
begin
|
||||
@@ -257,10 +256,10 @@ begin
|
||||
if NumDataPoints > 0 then
|
||||
begin
|
||||
DataIndexStart := 0;
|
||||
OpenPrice := ATickData[0].Ask;
|
||||
HighPrice := ATickData[0].Ask;
|
||||
LowPrice := ATickData[0].Ask;
|
||||
ClosePrice := ATickData[0].Ask;
|
||||
OpenPrice := ATickData[0].Data.Ask; // Changed
|
||||
HighPrice := ATickData[0].Data.Ask; // Changed
|
||||
LowPrice := ATickData[0].Data.Ask; // Changed
|
||||
ClosePrice := ATickData[0].Data.Ask; // Changed
|
||||
|
||||
for I := 0 to NumDataPoints - 1 do
|
||||
begin
|
||||
@@ -272,7 +271,7 @@ begin
|
||||
TempCandle.OpenPrice := OpenPrice;
|
||||
TempCandle.HighPrice := HighPrice;
|
||||
TempCandle.LowPrice := LowPrice;
|
||||
TempCandle.ClosePrice := ATickData[I - 1].Ask; // Close of previous candle is the tick before the new bar starts
|
||||
TempCandle.ClosePrice := ATickData[I - 1].Data.Ask; // Close of previous candle is the tick before the new bar starts // Changed
|
||||
TempCandle.OriginalDataIndexStart := DataIndexStart;
|
||||
TempCandle.OriginalDataIndexEnd := I - 1;
|
||||
TempCandle.TickVolume := (I - 1) - DataIndexStart + 1;
|
||||
@@ -280,27 +279,27 @@ begin
|
||||
|
||||
// Current tick 'I' starts a new candle
|
||||
DataIndexStart := I;
|
||||
OpenPrice := CurrentTick.Ask;
|
||||
HighPrice := CurrentTick.Ask;
|
||||
LowPrice := CurrentTick.Ask;
|
||||
ClosePrice := CurrentTick.Ask;
|
||||
OpenPrice := CurrentTick.Data.Ask; // Changed
|
||||
HighPrice := CurrentTick.Data.Ask; // Changed
|
||||
LowPrice := CurrentTick.Data.Ask; // Changed
|
||||
ClosePrice := CurrentTick.Data.Ask; // Changed
|
||||
end
|
||||
else // Current tick continues the existing candle OR is the first tick of the very first candle
|
||||
begin
|
||||
if I = DataIndexStart then // This is the first tick of the current (or overall first) candle
|
||||
begin
|
||||
OpenPrice := CurrentTick.Ask; // Open is set
|
||||
HighPrice := CurrentTick.Ask; // Initial High
|
||||
LowPrice := CurrentTick.Ask; // Initial Low
|
||||
OpenPrice := CurrentTick.Data.Ask; // Open is set // Changed
|
||||
HighPrice := CurrentTick.Data.Ask; // Initial High // Changed
|
||||
LowPrice := CurrentTick.Data.Ask; // Initial Low // Changed
|
||||
end
|
||||
else // Update High/Low for ongoing candle
|
||||
begin
|
||||
if CurrentTick.Ask > HighPrice then
|
||||
HighPrice := CurrentTick.Ask;
|
||||
if CurrentTick.Ask < LowPrice then
|
||||
LowPrice := CurrentTick.Ask;
|
||||
if CurrentTick.Data.Ask > HighPrice then // Changed
|
||||
HighPrice := CurrentTick.Data.Ask; // Changed
|
||||
if CurrentTick.Data.Ask < LowPrice then // Changed
|
||||
LowPrice := CurrentTick.Data.Ask; // Changed
|
||||
end;
|
||||
ClosePrice := CurrentTick.Ask; // Always update close to current tick
|
||||
ClosePrice := CurrentTick.Data.Ask; // Always update close to current tick // Changed
|
||||
end;
|
||||
|
||||
// If this is the last tick in the dataset, finalize the current candle
|
||||
@@ -379,15 +378,15 @@ begin
|
||||
Continue;
|
||||
end;
|
||||
|
||||
OpenPrice := ATickData[DataIndexStart].Ask;
|
||||
ClosePrice := ATickData[DataIndexEnd].Ask;
|
||||
OpenPrice := ATickData[DataIndexStart].Data.Ask; // Changed
|
||||
ClosePrice := ATickData[DataIndexEnd].Data.Ask; // Changed
|
||||
HighPrice := OpenPrice; // Initialize with Open
|
||||
LowPrice := OpenPrice; // Initialize with Open
|
||||
|
||||
for J := DataIndexStart to DataIndexEnd do
|
||||
begin
|
||||
if ATickData[J].Ask > HighPrice then HighPrice := ATickData[J].Ask;
|
||||
if ATickData[J].Ask < LowPrice then LowPrice := ATickData[J].Ask;
|
||||
if ATickData[J].Data.Ask > HighPrice then HighPrice := ATickData[J].Data.Ask; // Changed
|
||||
if ATickData[J].Data.Ask < LowPrice then LowPrice := ATickData[J].Data.Ask; // Changed
|
||||
end;
|
||||
// Ensure High/Low encompass Open/Close after loop
|
||||
HighPrice := Max(HighPrice, Max(OpenPrice, ClosePrice));
|
||||
@@ -409,10 +408,10 @@ begin
|
||||
SetLength(FCachedCandles, NumDataPoints);
|
||||
for I := 0 to NumDataPoints - 1 do
|
||||
begin
|
||||
FCachedCandles[I].OpenPrice := ATickData[I].Ask;
|
||||
FCachedCandles[I].HighPrice := ATickData[I].Ask;
|
||||
FCachedCandles[I].LowPrice := ATickData[I].Ask;
|
||||
FCachedCandles[I].ClosePrice := ATickData[I].Ask;
|
||||
FCachedCandles[I].OpenPrice := ATickData[I].Data.Ask; // Changed
|
||||
FCachedCandles[I].HighPrice := ATickData[I].Data.Ask; // Changed
|
||||
FCachedCandles[I].LowPrice := ATickData[I].Data.Ask; // Changed
|
||||
FCachedCandles[I].ClosePrice := ATickData[I].Data.Ask; // Changed
|
||||
FCachedCandles[I].OriginalDataIndexStart := I;
|
||||
FCachedCandles[I].OriginalDataIndexEnd := I;
|
||||
FCachedCandles[I].TickVolume := 1;
|
||||
@@ -438,10 +437,10 @@ begin
|
||||
BucketEndTimeOADate := BucketStartTimeOADate + TimeframeIntervalOADays;
|
||||
|
||||
DataIndexStart := CurrentTickIndex;
|
||||
OpenPrice := ATickData[CurrentTickIndex].Ask;
|
||||
HighPrice := ATickData[CurrentTickIndex].Ask;
|
||||
LowPrice := ATickData[CurrentTickIndex].Ask;
|
||||
ClosePrice := ATickData[CurrentTickIndex].Ask; // Initialize close with open
|
||||
OpenPrice := ATickData[CurrentTickIndex].Data.Ask; // Changed
|
||||
HighPrice := ATickData[CurrentTickIndex].Data.Ask; // Changed
|
||||
LowPrice := ATickData[CurrentTickIndex].Data.Ask; // Changed
|
||||
ClosePrice := ATickData[CurrentTickIndex].Data.Ask; // Initialize close with open // Changed
|
||||
DataIndexEnd := CurrentTickIndex; // Initialize end with start
|
||||
|
||||
// Iterate through ticks that fall into the current bucket
|
||||
@@ -449,9 +448,9 @@ begin
|
||||
(ATickData[CurrentTickIndex].OADateTime >= BucketStartTimeOADate) and
|
||||
(ATickData[CurrentTickIndex].OADateTime < BucketEndTimeOADate) do
|
||||
begin
|
||||
if ATickData[CurrentTickIndex].Ask > HighPrice then HighPrice := ATickData[CurrentTickIndex].Ask;
|
||||
if ATickData[CurrentTickIndex].Ask < LowPrice then LowPrice := ATickData[CurrentTickIndex].Ask;
|
||||
ClosePrice := ATickData[CurrentTickIndex].Ask; // Continuously update close price
|
||||
if ATickData[CurrentTickIndex].Data.Ask > HighPrice then HighPrice := ATickData[CurrentTickIndex].Data.Ask; // Changed
|
||||
if ATickData[CurrentTickIndex].Data.Ask < LowPrice then LowPrice := ATickData[CurrentTickIndex].Data.Ask; // Changed
|
||||
ClosePrice := ATickData[CurrentTickIndex].Data.Ask; // Continuously update close price // Changed
|
||||
DataIndexEnd := CurrentTickIndex; // Mark the last tick included in this candle
|
||||
Inc(CurrentTickIndex);
|
||||
end;
|
||||
|
||||
@@ -1,454 +0,0 @@
|
||||
unit Myc.TickLoader;
|
||||
|
||||
(* ------------------------------------------------------------------------------
|
||||
Unit: Myc.TickLoader
|
||||
Author: Delphi Algo Trading (Assistant)
|
||||
Date: May 27, 2025
|
||||
Description:
|
||||
This unit provides functions to load tick data (TTickData) from
|
||||
binary files. The data originates from a C# application
|
||||
and has the structure OADateTime (Double), Ask (Single), and Bid (Single).
|
||||
|
||||
The files follow the naming convention:
|
||||
For .tab files: Symbol_Year_MM.tab OR Symbol_Year_MM.tab_zip
|
||||
For .tab-live files: Symbol_Year_MM.tab-live (NEVER compressed)
|
||||
(MM represents the two-digit month, e.g., 01 for January, 12 for December)
|
||||
|
||||
If both compressed (.tab_zip) and uncompressed (.tab) versions of a .tab
|
||||
file exist, the compressed version is preferred. Files ending with _zip
|
||||
are expected to be ZIP archives containing the actual data file
|
||||
(e.g., Symbol_Year_MM.tab).
|
||||
|
||||
Loading occurs in two phases:
|
||||
1. All .tab files (regular or _zip) are processed chronologically by
|
||||
month and year. Unique ticks are loaded, and the timestamp of the
|
||||
latest tick across all .tab files is determined (overallLastTabTickTime).
|
||||
Assumes timestamps are unique across all .tab files.
|
||||
2. All .tab-live files (which are never compressed), for months/years
|
||||
corresponding to found .tab files, are processed. Ticks from a
|
||||
.tab-live file are only integrated if their timestamp is strictly later
|
||||
than overallLastTabTickTime. Assumes such ticks are globally unique.
|
||||
|
||||
If a .tab-live file results in no new data being added under these rules,
|
||||
it is deleted after processing.
|
||||
|
||||
The loading function automatically detects subsequent monthly/yearly files.
|
||||
|
||||
Main Function:
|
||||
LoadTickDataSeries - Loads a series of tick data files.
|
||||
------------------------------------------------------------------------------ *)
|
||||
|
||||
interface
|
||||
|
||||
uses
|
||||
System.SysUtils,
|
||||
System.Classes,
|
||||
System.Generics.Collections,
|
||||
System.IOUtils,
|
||||
Myc.Futures, // Added for TFuture in ReadFileTickData
|
||||
Myc.Signals; // Added for IMycState (dependency of TFuture)
|
||||
|
||||
type
|
||||
TTickData = packed record
|
||||
OADateTime: Double;
|
||||
Ask: Single;
|
||||
Bid: Single;
|
||||
end;
|
||||
|
||||
PTTickData = ^TTickData;
|
||||
|
||||
// TryParseFileName parses filenames like "Symbol_Year_MM.Extension" or "Symbol_Year_MM.Extension_zip"
|
||||
// to extract Path, Symbol, Year, and Month.
|
||||
function TryParseFileName( const FileName: string;
|
||||
out PathValue, SymbolValue: string;
|
||||
out YearValue, MonthValue: Integer ): Boolean;
|
||||
|
||||
function LoadTickDataSeries(const FileName: string): TFuture<TArray<TTickData>>;
|
||||
|
||||
// Asynchronously reads tick data from a single specified file.
|
||||
// Returns a TFuture that will provide an array of TTickData.
|
||||
function ReadFileTickData( var LoadGate: TLatch; const FileName: string ): TFuture<TArray<TTickData>>;
|
||||
|
||||
implementation
|
||||
|
||||
uses
|
||||
System.Zip,
|
||||
// Myc.Futures, Myc.Signals already in interface uses for TFuture type visibility
|
||||
Myc.TaskManager; // Added for the global TaskManager used by TFuture.Construct
|
||||
|
||||
function TryParseFileName( const FileName: string;
|
||||
out PathValue, SymbolValue: string;
|
||||
out YearValue, MonthValue: Integer ): Boolean;
|
||||
var
|
||||
fileNameNoPath: string;
|
||||
nameForParsing: string;
|
||||
parts: TArray<string>;
|
||||
begin
|
||||
Result := False; // Default to failure
|
||||
PathValue := ''; // Initialize all out parameters
|
||||
SymbolValue := '';
|
||||
YearValue := 0;
|
||||
MonthValue := 0;
|
||||
|
||||
PathValue := TPath.GetDirectoryName( FileName );
|
||||
fileNameNoPath := TPath.GetFileName( FileName );
|
||||
|
||||
nameForParsing := fileNameNoPath;
|
||||
if nameForParsing.EndsWith( '_zip', True ) then
|
||||
begin
|
||||
nameForParsing := nameForParsing.Substring( 0, nameForParsing.Length - 4 );
|
||||
end;
|
||||
|
||||
nameForParsing := TPath.GetFileNameWithoutExtension( nameForParsing ); // e.g., "EURUSD_2023_01"
|
||||
|
||||
parts := nameForParsing.Split( ['_'] );
|
||||
|
||||
if Length( parts ) < 3 then // Need at least Symbol_Year_Month
|
||||
begin
|
||||
exit;
|
||||
end;
|
||||
|
||||
// Month is the last part.
|
||||
if not TryStrToInt( parts[High( parts )], MonthValue ) then
|
||||
begin
|
||||
exit;
|
||||
end;
|
||||
if ( MonthValue < 1 ) or ( MonthValue > 12 ) then // Validate month range
|
||||
begin
|
||||
MonthValue := 0; // Set to invalid if out of range
|
||||
exit;
|
||||
end;
|
||||
|
||||
// Year is the second to last part.
|
||||
if not TryStrToInt( parts[High( parts ) - 1], YearValue ) then
|
||||
begin
|
||||
MonthValue := 0; // Also invalidate month if year parsing fails
|
||||
exit;
|
||||
end;
|
||||
|
||||
// Symbol consists of all parts before Year and Month. Count = Length(parts) - 2
|
||||
SymbolValue := string.Join( '_', Copy( parts, 0, Length( parts ) - 2 ) );
|
||||
|
||||
if SymbolValue = '' then // Ensure symbol is not empty
|
||||
begin
|
||||
exit;
|
||||
end;
|
||||
|
||||
Result := True;
|
||||
end;
|
||||
|
||||
function LoadTickDataSeries(const FileName: string): TFuture<TArray<TTickData>>;
|
||||
// Local type declaration for storing file information for phase 1 processing
|
||||
type
|
||||
TPhase1FileEntry = record
|
||||
Path: string;
|
||||
Year: Integer;
|
||||
Month: Integer;
|
||||
end;
|
||||
var
|
||||
initialYear, initialMonth: Integer;
|
||||
currentTabYear, currentTabMonth: Integer;
|
||||
pathName, symbolName: string;
|
||||
tabFileEntryRecord: TPhase1FileEntry; // Variable to construct record instances
|
||||
loadedState: TState;
|
||||
loadedFiles: TArray<TFuture<TArray<TTickData>>>;
|
||||
liveData: TFuture<TArray<TTickData>>;
|
||||
|
||||
begin // Start of LoadTickDataSeries
|
||||
if not TryParseFileName( FileName, pathName, symbolName, initialYear, initialMonth ) then
|
||||
begin
|
||||
raise EArgumentException.CreateFmt(
|
||||
'Invalid initial file name format: %s. Expected Symbol_Year_MM.Extension or Symbol_Year_MM.Extension_zip', [FileName] );
|
||||
end;
|
||||
|
||||
currentTabYear := initialYear;
|
||||
currentTabMonth := initialMonth;
|
||||
var tabBaseNameFormat: string;
|
||||
var compressedTabPath, uncompressedTabPath: string;
|
||||
var actualTabFileToLoad: string;
|
||||
var maxTabYearFound := -1;
|
||||
var maxTabMonthFound := -1;
|
||||
|
||||
var filesToLoadPhase1 := TList<TPhase1FileEntry>.Create; // Create list with the named record type
|
||||
try
|
||||
while True do
|
||||
begin
|
||||
tabBaseNameFormat := Format( '%s_%.4d_%.2d.tab', [symbolName, currentTabYear, currentTabMonth] );
|
||||
compressedTabPath := TPath.Combine( pathName, tabBaseNameFormat + '_zip' );
|
||||
uncompressedTabPath := TPath.Combine( pathName, tabBaseNameFormat );
|
||||
|
||||
actualTabFileToLoad := '';
|
||||
if TFile.Exists( compressedTabPath ) then
|
||||
begin
|
||||
actualTabFileToLoad := compressedTabPath;
|
||||
end
|
||||
else if TFile.Exists( uncompressedTabPath ) then
|
||||
begin
|
||||
actualTabFileToLoad := uncompressedTabPath;
|
||||
end;
|
||||
|
||||
if actualTabFileToLoad = '' then
|
||||
break; // No more sequential .tab files
|
||||
|
||||
// Construct the record and add it to the list
|
||||
tabFileEntryRecord.Path := actualTabFileToLoad;
|
||||
tabFileEntryRecord.Year := currentTabYear;
|
||||
tabFileEntryRecord.Month := currentTabMonth;
|
||||
filesToLoadPhase1.Add( tabFileEntryRecord );
|
||||
|
||||
// Advance to the next month
|
||||
Inc( currentTabMonth );
|
||||
if currentTabMonth > 12 then
|
||||
begin
|
||||
currentTabMonth := 1;
|
||||
Inc( currentTabYear );
|
||||
end;
|
||||
end;
|
||||
|
||||
// Now load and process identified .tab files
|
||||
var loadedFileList := TList<TFuture<TArray<TTickData>>>.Create;
|
||||
var loadStates := TList<TState>.Create;
|
||||
try
|
||||
var LoadGate: TLatch;
|
||||
|
||||
for var tabFileEntry: TPhase1FileEntry in filesToLoadPhase1 do // Iterate with the correct type
|
||||
begin
|
||||
var data := ReadFileTickData( LoadGate, tabFileEntry.Path );
|
||||
|
||||
loadedFileList.Add( TFuture<TArray<TTickData>>.Create( data ) );
|
||||
loadStates.Add( data.Done );
|
||||
|
||||
if tabFileEntry.Year > maxTabYearFound then
|
||||
begin
|
||||
maxTabYearFound := tabFileEntry.Year;
|
||||
maxTabMonthFound := tabFileEntry.Month;
|
||||
end
|
||||
else if ( tabFileEntry.Year = maxTabYearFound ) and ( tabFileEntry.Month > maxTabMonthFound ) then
|
||||
begin
|
||||
maxTabMonthFound := tabFileEntry.Month;
|
||||
end;
|
||||
end;
|
||||
|
||||
var liveFileBaseName := Format( '%s_%d_%02d.tab-live', [symbolName, maxTabYearFound, maxTabMonthFound] );
|
||||
liveData := ReadFileTickData( LoadGate, TPath.Combine( pathName, liveFileBaseName ) );
|
||||
loadStates.Add( liveData.Done );
|
||||
loadedFileList.Add( liveData );
|
||||
|
||||
loadedState := TState.All( loadStates.ToArray );
|
||||
loadedFiles := loadedFileList.ToArray;
|
||||
finally
|
||||
loadStates.Free;
|
||||
loadedFileList.Free;
|
||||
end;
|
||||
finally
|
||||
filesToLoadPhase1.Free;
|
||||
end;
|
||||
|
||||
Result := TFuture<TArray<TTickData>>.Construct(
|
||||
loadedState,
|
||||
function: TArray<TTickData>
|
||||
begin
|
||||
var tickList := TList<TTickData>.Create;
|
||||
try
|
||||
var totalTicks := Length(liveData.Result);
|
||||
var overallLastTabTickTime := 0.0;
|
||||
|
||||
for var P in loadedFiles do
|
||||
inc( totalTicks, Length(P.Result) );
|
||||
|
||||
tickList.Capacity := totalTicks;
|
||||
|
||||
for var P in loadedFiles do
|
||||
begin
|
||||
if Length(P.Result) > 0 then
|
||||
begin
|
||||
inc( totalTicks, Length(P.Result) );
|
||||
|
||||
for var iterTickData in P.Result do
|
||||
if iterTickData.OADateTime > overallLastTabTickTime then
|
||||
begin
|
||||
tickList.Add( iterTickData );
|
||||
overallLastTabTickTime := iterTickData.OADateTime;
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
Result := tickList.ToArray;
|
||||
finally
|
||||
tickList.Free;
|
||||
end;
|
||||
end );
|
||||
end;
|
||||
|
||||
function LoadSingleFileTickDataFromStream(const InputStream: TStream): TArray<TTickData>;
|
||||
var
|
||||
fileSize: Int64;
|
||||
recordCount: Integer;
|
||||
bytesRead: Integer;
|
||||
begin
|
||||
// Initialize Result record
|
||||
SetLength(Result, 0);
|
||||
|
||||
InputStream.Position := 0;
|
||||
|
||||
fileSize := InputStream.Size;
|
||||
if fileSize = 0 then
|
||||
exit;
|
||||
|
||||
if (fileSize mod SizeOf(TTickData)) <> 0 then
|
||||
raise EReadError.CreateFmt('File corrupted. Size %d is not a multiple of record size %d.',
|
||||
[fileSize, SizeOf(TTickData)]);
|
||||
|
||||
recordCount := fileSize div SizeOf(TTickData);
|
||||
if recordCount > 0 then
|
||||
begin
|
||||
SetLength(Result, recordCount);
|
||||
bytesRead := InputStream.Read(Result[0], fileSize);
|
||||
if bytesRead <> fileSize then
|
||||
raise EReadError.CreateFmt('Read error. Expected to read %d bytes, but actually read %d bytes.',
|
||||
[fileSize, bytesRead]);
|
||||
end;
|
||||
end;
|
||||
|
||||
function LoadUncompressedSingleFileTickData(const FullFileName: string): TArray<TTickData>;
|
||||
var
|
||||
inputStream: TStream;
|
||||
begin
|
||||
// Initialize Result record
|
||||
SetLength(Result, 0);
|
||||
|
||||
if not TFile.Exists(FullFileName) then
|
||||
begin
|
||||
exit;
|
||||
end;
|
||||
|
||||
inputStream := TFileStream.Create(FullFileName, fmOpenRead or fmShareDenyWrite);
|
||||
try
|
||||
try
|
||||
Result := LoadSingleFileTickDataFromStream( inputStream );
|
||||
|
||||
except
|
||||
on E: EReadError do
|
||||
begin
|
||||
raise EReadError.CreateFmt('File %s: %s', [FullFileName, E.Message]);
|
||||
end;
|
||||
end;
|
||||
finally
|
||||
inputStream.Free;
|
||||
end;
|
||||
end;
|
||||
|
||||
function UncompressTickDataFromStream(const InputStream: TStream): TArray<TTickData>;
|
||||
var
|
||||
decompressionStream: TStream; // Holds uncompressed data of a single zip entry
|
||||
localHeader: TZipHeader;
|
||||
entryIndex: Integer;
|
||||
i: Integer;
|
||||
zipFileInstance: TZipFile;
|
||||
begin
|
||||
// Initialize Result record
|
||||
SetLength(Result, 0);
|
||||
|
||||
decompressionStream := nil;
|
||||
zipFileInstance := nil;
|
||||
|
||||
try
|
||||
InputStream.Position := 0; // Reset stream position to the beginning for TZipFile
|
||||
|
||||
// Open TZipFile from this memory stream
|
||||
zipFileInstance := TZipFile.Create;
|
||||
zipFileInstance.Open(InputStream, TZipMode.zmRead); // Open the zip from the memory stream
|
||||
|
||||
if zipFileInstance.FileCount = 0 then
|
||||
begin
|
||||
// If TZipFile opens an empty or invalid stream successfully but finds no files
|
||||
exit; // FileSuccessfullyLoaded remains False
|
||||
end;
|
||||
|
||||
entryIndex := -1;
|
||||
for i := 0 to zipFileInstance.FileCount - 1 do
|
||||
begin
|
||||
if zipFileInstance.FileNames[i].EndsWith('.tab', True) then
|
||||
begin
|
||||
entryIndex := i;
|
||||
break;
|
||||
end;
|
||||
end;
|
||||
|
||||
if entryIndex = -1 then // If specific name not found, default to the first entry
|
||||
begin
|
||||
if zipFileInstance.FileCount > 0 then
|
||||
entryIndex := 0
|
||||
else
|
||||
exit; // Should not happen if FileCount > 0 check passed
|
||||
end;
|
||||
|
||||
// Decompress the specific entry from the in-memory zip file.
|
||||
// TZipFile.Read creates and populates decompressionStream with uncompressed data.
|
||||
zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
|
||||
|
||||
if not Assigned(decompressionStream) then
|
||||
exit; // Exit if no valid stream could be prepared
|
||||
|
||||
Result := LoadSingleFileTickDataFromStream( decompressionStream );
|
||||
finally
|
||||
// Ensure all potentially created objects are freed
|
||||
if Assigned(decompressionStream) then // Holds uncompressed data from a zip entry
|
||||
decompressionStream.Free;
|
||||
if Assigned(zipFileInstance) then
|
||||
zipFileInstance.Free;
|
||||
end;
|
||||
end;
|
||||
|
||||
function ReadFileTickData( var LoadGate: TLatch; const FileName: string ): TFuture<TArray<TTickData>>;
|
||||
var
|
||||
parsedPath, parsedSymbol: string;
|
||||
parsedYear, parsedMonth: Integer;
|
||||
begin
|
||||
Result := TFuture<TArray<TTickData>>.Null;
|
||||
|
||||
// Attempt to parse year and month from the filename
|
||||
if not TryParseFileName( FileName, parsedPath, parsedSymbol, parsedYear, parsedMonth ) then
|
||||
begin
|
||||
// If filename parsing fails, return a future that resolves to an empty array.
|
||||
exit;
|
||||
end;
|
||||
|
||||
if FileName.EndsWith('_zip', True) then
|
||||
begin
|
||||
var zipFileBytes := TFile.ReadAllBytes(FileName); // Read all bytes of the .zip file
|
||||
if Length(zipFileBytes) = 0 then // Handle empty zip file case
|
||||
exit;
|
||||
|
||||
var capFileName := FileName;
|
||||
var blob := TFuture<TBytes>.Construct(
|
||||
TLatch.Enqueue( LoadGate ),
|
||||
function: TBytes
|
||||
begin
|
||||
Result := TFile.ReadAllBytes(capFileName); // Read all bytes of the .zip file
|
||||
end );
|
||||
|
||||
Result := blob.Chain<TArray<TTickData>>(
|
||||
function( bytes: TBytes ): TArray<TTickData>
|
||||
begin
|
||||
var zipMemoryStream := TBytesStream.Create( zipFileBytes ); // Create a memory stream to hold the zip file content
|
||||
try
|
||||
zipMemoryStream.Position := 0;
|
||||
Result := UncompressTickDataFromStream( zipMemoryStream );
|
||||
finally
|
||||
zipMemoryStream.Free;
|
||||
end;
|
||||
end );
|
||||
end
|
||||
else
|
||||
begin
|
||||
var capFileName := FileName;
|
||||
Result := TFuture<TArray<TTickData>>.Construct(
|
||||
function: TArray<TTickData>
|
||||
begin
|
||||
Result := LoadUncompressedSingleFileTickData( capFileName );
|
||||
end );
|
||||
end;
|
||||
end;
|
||||
|
||||
end.
|
||||
Reference in New Issue
Block a user