Removed bottleneck in DataStream
This commit is contained in:
Michael Schimmel
+53
-35
@@ -2,6 +2,8 @@ unit MainForm;
|
||||
|
||||
interface
|
||||
|
||||
{.$define TICKDATA}
|
||||
|
||||
uses
|
||||
System.SysUtils,
|
||||
System.Types,
|
||||
@@ -88,14 +90,22 @@ type
|
||||
private
|
||||
FOnEvent: TNotifyEvent;
|
||||
{ Private declarations }
|
||||
{$ifdef TICKDATA}
|
||||
FServer: IDataServer<TAuraAskBidFileItem>;
|
||||
{$else}
|
||||
FServer: IDataServer<TOhlcItem>;
|
||||
{$endif}
|
||||
FSymbols: TFuture<TArray<String>>;
|
||||
FTerminate: TEvent;
|
||||
FProcessDone: TState;
|
||||
FApplication: IAuraApplication;
|
||||
FModulesItem: TTreeViewItem;
|
||||
function SelectedSymbol: String;
|
||||
function ExecuteStrategy(const Symbol: String; const Processor: IMycProcessor<TArray<TDataPoint<TAskBidItem>>>): TState;
|
||||
function ExecuteStrategy(
|
||||
const Symbol: String;
|
||||
Timeframe: TTimeframe;
|
||||
const Processor: IMycProcessor<TDataPoint<TOhlcItem>>
|
||||
): TState;
|
||||
|
||||
public
|
||||
procedure NewWorkspace;
|
||||
@@ -215,7 +225,11 @@ begin
|
||||
FTerminate := TEvent.CreateEvent;
|
||||
|
||||
// Create an instance of the TAuraTABFileServer. The server can be reused for multiple stream creations. [364]
|
||||
{$ifdef TICKDATA}
|
||||
FServer := TAuraTABFileServer.Create('\\COFFEE\TickData\Pepperstone');
|
||||
{$else}
|
||||
FServer := TAuraM1FileServer.Create('\\COFFEE\TickData\Pepperstone');
|
||||
{$endif}
|
||||
|
||||
SymbolsComboBox.Enabled := false;
|
||||
ChartButton.Enabled := false;
|
||||
@@ -299,13 +313,27 @@ begin
|
||||
Result := Res;
|
||||
end;
|
||||
|
||||
function TForm1.ExecuteStrategy(const Symbol: String; const Processor: IMycProcessor<TArray<TDataPoint<TAskBidItem>>>): TState;
|
||||
var
|
||||
dataProvider: TConverter<TArray<TDataPoint<TAuraAskBidFileItem>>, TArray<TDataPoint<TAskBidItem>>>;
|
||||
function TForm1.ExecuteStrategy(const Symbol: String; Timeframe: TTimeframe; const Processor: IMycProcessor<TDataPoint<TOhlcItem>>): TState;
|
||||
begin
|
||||
var terminated := TFlag.CreateObserver(FTerminate.Signal).State;
|
||||
|
||||
dataProvider :=
|
||||
{$ifdef TICKDATA}
|
||||
var ticker := TConverter.CreateTicker<TDataPoint<TAskBidItem>>;
|
||||
|
||||
var lastPrice :=
|
||||
ticker.Chain<TDataPoint<Double>>(
|
||||
function(const Tick: TDataPoint<TAskBidItem>): TDataPoint<Double>
|
||||
begin
|
||||
Result.Time := Tick.Time;
|
||||
Result.Data := 0.5 * (Tick.Data.Ask + Tick.Data.Bid);
|
||||
end
|
||||
);
|
||||
|
||||
var OhlcPoint := lastPrice.Chain<TDataPoint<TOhlcItem>>(TConverter.CreateTickAggregation(Timeframe));
|
||||
|
||||
OhlcPoint.Sender.Link(Processor);
|
||||
|
||||
var dataProvider :=
|
||||
TConverter<TArray<TDataPoint<TAuraAskBidFileItem>>, TArray<TDataPoint<TAskBidItem>>>.CreateGeneric(
|
||||
function(const Values: TArray<TDataPoint<TAuraAskBidFileItem>>): TArray<TDataPoint<TAskBidItem>>
|
||||
begin
|
||||
@@ -319,9 +347,20 @@ begin
|
||||
end
|
||||
);
|
||||
|
||||
dataProvider.Sender.Link(Processor);
|
||||
dataProvider.Sender.Link(ticker);
|
||||
|
||||
Result := FServer.ProcessData(Symbol, terminated, dataProvider);
|
||||
{$else}
|
||||
|
||||
var ticker := TConverter.CreateTicker<TDataPoint<TOhlcItem>>;
|
||||
|
||||
var OhlcPoint := ticker.Chain<TDataPoint<TOhlcItem>>(TConverter.CreateOhlcAggregation(Timeframe));
|
||||
|
||||
OhlcPoint.Sender.Link(Processor);
|
||||
|
||||
Result := FServer.ProcessData(Symbol, terminated, ticker);
|
||||
|
||||
{$endif}
|
||||
end;
|
||||
|
||||
function TForm1.SelectedSymbol: String;
|
||||
@@ -350,20 +389,9 @@ begin
|
||||
|
||||
/////
|
||||
|
||||
var timeframe := TTimeframe.M15;
|
||||
var OhlcPoint := TConverter.CreateIdentity<TDataPoint<TOhlcItem>>;
|
||||
|
||||
var ticker := TConverter.CreateTicker<TDataPoint<TAskBidItem>>;
|
||||
|
||||
var lastPrice :=
|
||||
ticker.Chain<TDataPoint<Double>>(
|
||||
function(const Tick: TDataPoint<TAskBidItem>): TDataPoint<Double>
|
||||
begin
|
||||
Result.Time := Tick.Time;
|
||||
Result.Data := 0.5 * (Tick.Data.Ask + Tick.Data.Bid);
|
||||
end
|
||||
);
|
||||
|
||||
var OhlcPoint := lastPrice.Chain<TDataPoint<TOhlcItem>>(TConverter.CreateAggregation(timeframe));
|
||||
var timeframe := TTimeframe.H;
|
||||
|
||||
var Timestamps := OhlcPoint.Field<TDateTime>('Time');
|
||||
var Ohlc := OhlcPoint.Field<TOhlcItem>('Data');
|
||||
@@ -425,7 +453,7 @@ begin
|
||||
}
|
||||
/////
|
||||
|
||||
var done := ExecuteStrategy(Symbol, ticker);
|
||||
var done := ExecuteStrategy(Symbol, timeframe, OhlcPoint);
|
||||
FProcessDone := TState.All([FProcessDone, done]);
|
||||
end;
|
||||
|
||||
@@ -437,19 +465,12 @@ type
|
||||
Entry: Double;
|
||||
pnl: Double;
|
||||
end;
|
||||
var
|
||||
panel: TMycChart.TPanel;
|
||||
begin
|
||||
var timeframe := TTimeframe.M15;
|
||||
|
||||
var ticker := TConverter.CreateTicker<TDataPoint<TAskBidItem>>;
|
||||
|
||||
var lastPrice :=
|
||||
ticker.Chain<TDataPoint<Double>>(
|
||||
TConverter.CreateDataPointConverter<TAskBidItem, Double>(
|
||||
function(const Tick: TAskBidItem): Double begin Result := 0.5 * (Tick.Ask + Tick.Bid); end
|
||||
)
|
||||
);
|
||||
|
||||
var OhlcPoint := lastPrice.Chain<TDataPoint<TOhlcItem>>(TConverter.CreateAggregation(timeframe)).MakeParallel;
|
||||
var OhlcPoint := TConverter.CreateIdentity<TDataPoint<TOhlcItem>>;
|
||||
|
||||
var Ohlc := TConverter.CreateSequence<TOhlcItem>(2, OhlcPoint.Field<TOhlcItem>('Data').Sender);
|
||||
|
||||
@@ -575,16 +596,13 @@ begin
|
||||
|
||||
chart.SetXAxisSeries(M15, OhlcPoint.Field<TDateTime>('Time').Sender);
|
||||
|
||||
var panel := chart.AddPanel;
|
||||
panel := chart.AddPanel;
|
||||
panel.AddOhlcSeries(Ohlc[0].Sender);
|
||||
panel.AddDoubleSeries(Hull.Sender, TAlphaColors.Cornflowerblue, 2);
|
||||
panel.AddDoubleSeries(Sma.Sender, TAlphaColors.Brown, 1.5);
|
||||
panel.AddDoubleSeries(Signal.Field<Double>('SL').Sender, TAlphaColors.Red, 2);
|
||||
panel.AddDoubleSeries(Signal.Field<Double>('Entry').Sender, TAlphaColors.Green, 1);
|
||||
|
||||
// panel := chart.AddPanel;
|
||||
// panel.AddDoubleSeries( equity.Sender, TAlphaColors.Blue, 3 );
|
||||
|
||||
var pnlChart := TMycChart.Create(Self);
|
||||
AlignControl(pnlChart);
|
||||
pnlChart.Height := Layout.ChildrenRect.Width * 9 / 24;
|
||||
@@ -597,7 +615,7 @@ begin
|
||||
|
||||
/////
|
||||
|
||||
var done := ExecuteStrategy(Symbol, ticker);
|
||||
var done := ExecuteStrategy(Symbol, timeframe, OhlcPoint);
|
||||
FProcessDone := TState.All([FProcessDone, done]);
|
||||
end;
|
||||
|
||||
|
||||
@@ -158,6 +158,12 @@ type
|
||||
|
||||
// Endpoint that collects data into a series.
|
||||
TMycDataEndpoint<T> = class(TInterfacedObject, TMutable<TSeries<T>>.IMutable)
|
||||
type
|
||||
PItem = ^TItem;
|
||||
TItem = record
|
||||
Next: PItem;
|
||||
Value: T;
|
||||
end;
|
||||
private
|
||||
FProcessor: TMycContainedProcessor<T>;
|
||||
FTag: TDataProvider<T>.TTag;
|
||||
@@ -166,6 +172,7 @@ type
|
||||
FData: TSeries<T>;
|
||||
FChanged: TEvent;
|
||||
FLock: TLightweightMREW;
|
||||
FFirst: PItem;
|
||||
function GetChanged: TSignal;
|
||||
function GetValue: TSeries<T>;
|
||||
function ProcessData(const Value: T): TState;
|
||||
@@ -195,6 +202,20 @@ type
|
||||
function ProcessData(const Value: T): TState; override; final;
|
||||
end;
|
||||
|
||||
TOhlcAggregation = class(TMycConverter<TDataPoint<TOhlcItem>, TDataPoint<TOhlcItem>>)
|
||||
private
|
||||
FTimeframe: TTimeframe;
|
||||
FCurrentBar: TDataPoint<TOhlcItem>;
|
||||
function GetBarStartTime(const TimeStamp: TDateTime; const Timeframe: TTimeframe): TDateTime;
|
||||
function GetCurrentBar: TDataPoint<TOhlcItem>;
|
||||
function GetTimeframe: TTimeframe;
|
||||
public
|
||||
constructor Create(const ATimeframe: TTimeframe);
|
||||
function ProcessData(const Value: TDataPoint<TOhlcItem>): TState; override;
|
||||
property CurrentBar: TDataPoint<TOhlcItem> read GetCurrentBar;
|
||||
property Timeframe: TTimeframe read GetTimeframe;
|
||||
end;
|
||||
|
||||
implementation
|
||||
|
||||
uses
|
||||
@@ -461,11 +482,42 @@ end;
|
||||
|
||||
function TMycDataEndpoint<T>.GetValue: TSeries<T>;
|
||||
begin
|
||||
FLock.BeginRead;
|
||||
FLock.BeginWrite;
|
||||
try
|
||||
var cnt := 0;
|
||||
var tmp: PItem := nil;
|
||||
var item: PItem;
|
||||
|
||||
while FFirst <> nil do
|
||||
begin
|
||||
item := FFirst;
|
||||
FFirst := item.Next;
|
||||
item.Next := tmp;
|
||||
tmp := item;
|
||||
inc(cnt);
|
||||
end;
|
||||
|
||||
if cnt > 0 then
|
||||
begin
|
||||
var Arr: TArray<T>;
|
||||
SetLength(Arr, cnt);
|
||||
|
||||
cnt := 0;
|
||||
while tmp <> nil do
|
||||
begin
|
||||
item := tmp;
|
||||
tmp := item.Next;
|
||||
Arr[cnt] := item.Value;
|
||||
inc(cnt);
|
||||
Dispose(item);
|
||||
end;
|
||||
|
||||
FData := FData.Add(Arr, 0, cnt, FLookback);
|
||||
end;
|
||||
|
||||
Result := FData;
|
||||
finally
|
||||
FLock.EndRead;
|
||||
FLock.EndWrite;
|
||||
end;
|
||||
end;
|
||||
|
||||
@@ -474,8 +526,14 @@ begin
|
||||
Result := TState.Null;
|
||||
FLock.BeginWrite;
|
||||
try
|
||||
var P: PItem;
|
||||
New(P);
|
||||
P.Next := FFirst;
|
||||
FFirst := P;
|
||||
P.Value := Value;
|
||||
|
||||
// Add new data point, respecting the lookback period.
|
||||
FData := FData.Add(Value, FLookback);
|
||||
// FData := FData.Add(Value, FLookback);
|
||||
finally
|
||||
FLock.EndWrite;
|
||||
end;
|
||||
@@ -657,4 +715,104 @@ begin
|
||||
FQueue := Result;
|
||||
end;
|
||||
|
||||
{ TOhlcAggregation }
|
||||
|
||||
constructor TOhlcAggregation.Create(const ATimeframe: TTimeframe);
|
||||
begin
|
||||
inherited Create;
|
||||
FTimeframe := ATimeframe;
|
||||
end;
|
||||
|
||||
function TOhlcAggregation.GetBarStartTime(const TimeStamp: TDateTime; const Timeframe: TTimeframe): TDateTime;
|
||||
var
|
||||
baseTime: TDateTime;
|
||||
begin
|
||||
// Align the time grid to UTC 0:00 using functions from System.DateUtils
|
||||
baseTime := RecodeMilliSecond(TimeStamp, 0);
|
||||
|
||||
case Timeframe of
|
||||
S: Result := baseTime;
|
||||
S5: Result := RecodeSecond(baseTime, SecondOf(TimeStamp) - (SecondOf(TimeStamp) mod 5));
|
||||
S15: Result := RecodeSecond(baseTime, SecondOf(TimeStamp) - (SecondOf(TimeStamp) mod 15));
|
||||
S30: Result := RecodeSecond(baseTime, SecondOf(TimeStamp) - (SecondOf(TimeStamp) mod 30));
|
||||
|
||||
M: Result := RecodeSecond(baseTime, 0);
|
||||
M2: Result := RecodeMinute(RecodeSecond(baseTime, 0), MinuteOf(TimeStamp) - (MinuteOf(TimeStamp) mod 2));
|
||||
M3: Result := RecodeMinute(RecodeSecond(baseTime, 0), MinuteOf(TimeStamp) - (MinuteOf(TimeStamp) mod 3));
|
||||
M5: Result := RecodeMinute(RecodeSecond(baseTime, 0), MinuteOf(TimeStamp) - (MinuteOf(TimeStamp) mod 5));
|
||||
M10: Result := RecodeMinute(RecodeSecond(baseTime, 0), MinuteOf(TimeStamp) - (MinuteOf(TimeStamp) mod 10));
|
||||
M15: Result := RecodeMinute(RecodeSecond(baseTime, 0), MinuteOf(TimeStamp) - (MinuteOf(TimeStamp) mod 15));
|
||||
M30: Result := RecodeMinute(RecodeSecond(baseTime, 0), MinuteOf(TimeStamp) - (MinuteOf(TimeStamp) mod 30));
|
||||
|
||||
H: Result := RecodeMinute(RecodeSecond(baseTime, 0), 0);
|
||||
H2: Result := RecodeHour(RecodeMinute(RecodeSecond(baseTime, 0), 0), HourOf(TimeStamp) - (HourOf(TimeStamp) mod 2));
|
||||
H3: Result := RecodeHour(RecodeMinute(RecodeSecond(baseTime, 0), 0), HourOf(TimeStamp) - (HourOf(TimeStamp) mod 3));
|
||||
H4: Result := RecodeHour(RecodeMinute(RecodeSecond(baseTime, 0), 0), HourOf(TimeStamp) - (HourOf(TimeStamp) mod 4));
|
||||
H8: Result := RecodeHour(RecodeMinute(RecodeSecond(baseTime, 0), 0), HourOf(TimeStamp) - (HourOf(TimeStamp) mod 8));
|
||||
H12: Result := RecodeHour(RecodeMinute(RecodeSecond(baseTime, 0), 0), HourOf(TimeStamp) - (HourOf(TimeStamp) mod 12));
|
||||
|
||||
D: Result := StartOfTheDay(TimeStamp);
|
||||
// D2, D3 are uncommon; this is a simple modulo-based approach relative to TDateTime's epoch.
|
||||
D2: Result := Floor(TimeStamp) - (Floor(TimeStamp) mod 2);
|
||||
D3: Result := Floor(TimeStamp) - (Floor(TimeStamp) mod 3);
|
||||
|
||||
W: Result := TimeStamp.StartOfTheWeek;
|
||||
|
||||
MN: Result := TimeStamp.StartOfTheMonth;
|
||||
// Quarter alignment
|
||||
MN3: Result := RecodeMonth(TimeStamp.StartOfTheMonth, (MonthOf(TimeStamp) - 1) div 3 * 3 + 1);
|
||||
// Half-year alignment
|
||||
MN6: Result := RecodeMonth(TimeStamp.StartOfTheMonth, (MonthOf(TimeStamp) - 1) div 6 * 6 + 1);
|
||||
|
||||
Y: Result := TimeStamp.StartOfTheYear;
|
||||
else
|
||||
// Fallback for any undefined timeframe
|
||||
Result := 0;
|
||||
end;
|
||||
end;
|
||||
|
||||
function TOhlcAggregation.GetCurrentBar: TDataPoint<TOhlcItem>;
|
||||
begin
|
||||
Result := FCurrentBar;
|
||||
end;
|
||||
|
||||
function TOhlcAggregation.GetTimeframe: TTimeframe;
|
||||
begin
|
||||
Result := FTimeframe;
|
||||
end;
|
||||
|
||||
function TOhlcAggregation.ProcessData(const Value: TDataPoint<TOhlcItem>): TState;
|
||||
var
|
||||
barStartTime: TDateTime;
|
||||
lastBarTime: TDateTime;
|
||||
begin
|
||||
// Update bar for the strategy's timeframe
|
||||
barStartTime := GetBarStartTime(Value.Time, FTimeframe);
|
||||
lastBarTime := FCurrentBar.Time;
|
||||
|
||||
if (barStartTime > lastBarTime) then
|
||||
begin
|
||||
// A new bar starts, so the previous one is now complete.
|
||||
if (lastBarTime > 0) then
|
||||
begin
|
||||
Result := Broadcast(FCurrentBar);
|
||||
end;
|
||||
|
||||
// Start a new bar, Volume is 1 because this is the first tick.
|
||||
FCurrentBar.Data := Value.Data;
|
||||
FCurrentBar.Time := barStartTime;
|
||||
end
|
||||
else
|
||||
begin
|
||||
// Update the currently aggregating bar
|
||||
if Value.Data.High > FCurrentBar.Data.High then
|
||||
FCurrentBar.Data.High := Value.Data.High;
|
||||
if Value.Data.Low < FCurrentBar.Data.Low then
|
||||
FCurrentBar.Data.Low := Value.Data.Low;
|
||||
FCurrentBar.Data.Close := Value.Data.Close;
|
||||
// Volume is the number of ticks needed to build the complete bar.
|
||||
FCurrentBar.Data.Volume := FCurrentBar.Data.Volume + Value.Data.Volume;
|
||||
end;
|
||||
end;
|
||||
|
||||
end.
|
||||
|
||||
@@ -111,7 +111,8 @@ type
|
||||
class function CreateRecordField<S, T>(const FieldName: String): TConverter<S, T>; static;
|
||||
class function CreateIdentity<T>: TConverter<T, T>; static;
|
||||
|
||||
class function CreateAggregation(Timeframe: TTimeframe): TConverter<TDataPoint<Double>, TDataPoint<TOhlcItem>>; static;
|
||||
class function CreateTickAggregation(Timeframe: TTimeframe): TConverter<TDataPoint<Double>, TDataPoint<TOhlcItem>>; static;
|
||||
class function CreateOhlcAggregation(Timeframe: TTimeframe): TConverter<TDataPoint<TOhlcItem>, TDataPoint<TOhlcItem>>; static;
|
||||
|
||||
class function CreateDataPointConverter<S, T>(const Func: TConstFunc<S, T>): TConverter<TDataPoint<S>, TDataPoint<T>>; static;
|
||||
|
||||
@@ -251,7 +252,7 @@ begin
|
||||
Result := A.FConverter;
|
||||
end;
|
||||
|
||||
class function TConverter.CreateAggregation(Timeframe: TTimeframe): TConverter<TDataPoint<Double>, TDataPoint<TOhlcItem>>;
|
||||
class function TConverter.CreateTickAggregation(Timeframe: TTimeframe): TConverter<TDataPoint<Double>, TDataPoint<TOhlcItem>>;
|
||||
begin
|
||||
Result := TTickAggregation.Create(Timeframe);
|
||||
end;
|
||||
@@ -309,6 +310,11 @@ begin
|
||||
Parent.Link(seq);
|
||||
end;
|
||||
|
||||
class function TConverter.CreateOhlcAggregation(Timeframe: TTimeframe): TConverter<TDataPoint<TOhlcItem>, TDataPoint<TOhlcItem>>;
|
||||
begin
|
||||
Result := TOhlcAggregation.Create(Timeframe);
|
||||
end;
|
||||
|
||||
class function TConverter.Parallel<T>(Parent: TDataProvider<T>): TConverter<T, T>;
|
||||
begin
|
||||
Result := TMycParallelConverter<T>.Create;
|
||||
|
||||
+356
-207
@@ -34,13 +34,12 @@ type
|
||||
FSymbol: String;
|
||||
FYear: Integer;
|
||||
FMonth: Integer;
|
||||
FBasename: String;
|
||||
function GetIsValid: Boolean;
|
||||
public
|
||||
constructor Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer);
|
||||
constructor Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer; const ABasename: String);
|
||||
function GetBaseFileName: string;
|
||||
function GetFullFileName: string;
|
||||
// Gets the next consecutive data file, if it exists on disk.
|
||||
function GetNextFile: TAuraDataFile;
|
||||
property IsValid: Boolean read GetIsValid;
|
||||
property Extension: String read FExtension;
|
||||
property Path: String read FPath;
|
||||
@@ -54,12 +53,10 @@ type
|
||||
private
|
||||
FCachedFiles: TDataFileCache<TArray<TArray<TDataPoint<T>>>>;
|
||||
FPath: String;
|
||||
FSymbols: TFuture<TDictionary<String, TAuraDataFile>>;
|
||||
// The cache now holds a sorted array of all files for each symbol.
|
||||
FSymbols: TFuture<TDictionary<String, TArray<TAuraDataFile>>>;
|
||||
function GetPath: String;
|
||||
|
||||
// Used by cache to actually load an uncached file.
|
||||
function DoLoad(const FileName: string): TFuture<TArray<TArray<TDataPoint<T>>>>;
|
||||
|
||||
function ProcessChunks(
|
||||
const DataChunks: TArray<TArray<TDataPoint<T>>>;
|
||||
Terminated: TState;
|
||||
@@ -78,10 +75,10 @@ type
|
||||
FLoadGate: TLatch;
|
||||
|
||||
protected
|
||||
// Read data from stream and return it chunked.
|
||||
class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<T>>>; virtual; abstract;
|
||||
// Read data from stream and return it chunked.
|
||||
class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<T>>>; virtual; abstract;
|
||||
// Used by cache to actually load an uncached file. Now virtual and abstract.
|
||||
function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<T>>>>; virtual; abstract;
|
||||
// Gets the next consecutive data file from the cached, sorted list.
|
||||
function GetNextFile(const Curr: TAuraDataFile): TAuraDataFile; virtual;
|
||||
|
||||
public
|
||||
constructor Create(const APath: String);
|
||||
@@ -110,12 +107,36 @@ type
|
||||
end;
|
||||
|
||||
TAuraTABFileServer = class(TAuraDataServer<TAuraAskBidFileItem>)
|
||||
private
|
||||
class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>;
|
||||
class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>;
|
||||
protected
|
||||
// Scans a directory and returns the oldest file found for each symbol.
|
||||
class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>; override;
|
||||
class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>; override;
|
||||
function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>; override;
|
||||
public
|
||||
function ParseFileName(const FileName: string): TAuraDataFile; override;
|
||||
end;
|
||||
|
||||
// File record for cTrader M1 data
|
||||
TAuraM1FileItem = packed record
|
||||
OADateTime: Double;
|
||||
Open: Int64;
|
||||
High: Int64;
|
||||
Low: Int64;
|
||||
Close: Int64;
|
||||
Spread: Int64;
|
||||
TickVolume: Integer;
|
||||
Digits: Integer;
|
||||
end;
|
||||
|
||||
// File server for cTrader M1 data. Note that the generic type is TOhlcItem,
|
||||
// as the server converts the file data into standard OHLC items.
|
||||
TAuraM1FileServer = class(TAuraDataServer<TOhlcItem>)
|
||||
private
|
||||
class function ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
|
||||
class function ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
|
||||
protected
|
||||
function DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>; override;
|
||||
public
|
||||
function LoadDataSeries(const InitialFile: TAuraDataFile): TFuture<TArray<TDataPoint<TAuraAskBidFileItem>>>;
|
||||
function ParseFileName(const FileName: string): TAuraDataFile; override;
|
||||
end;
|
||||
|
||||
@@ -123,6 +144,7 @@ implementation
|
||||
|
||||
uses
|
||||
System.Zip,
|
||||
System.ZLib,
|
||||
System.Math,
|
||||
System.StrUtils,
|
||||
Myc.TaskManager;
|
||||
@@ -133,18 +155,19 @@ const
|
||||
|
||||
{ TAuraDataFile }
|
||||
|
||||
constructor TAuraDataFile.Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer);
|
||||
constructor TAuraDataFile.Create(const APath, ASymbol, AExtension: String; AYear, AMonth: Integer; const ABasename: String);
|
||||
begin
|
||||
FPath := APath;
|
||||
FSymbol := ASymbol;
|
||||
FExtension := AExtension;
|
||||
FYear := AYear;
|
||||
FMonth := AMonth;
|
||||
FBasename := ABasename;
|
||||
end;
|
||||
|
||||
function TAuraDataFile.GetBaseFileName: string;
|
||||
begin
|
||||
Result := Format('%s_%.4d_%.2d', [FSymbol, FYear, FMonth]);
|
||||
Result := FBasename;
|
||||
end;
|
||||
|
||||
function TAuraDataFile.GetFullFileName: string;
|
||||
@@ -157,36 +180,6 @@ begin
|
||||
Result := (FSymbol <> '') and (FYear > 0);
|
||||
end;
|
||||
|
||||
function TAuraDataFile.GetNextFile: TAuraDataFile;
|
||||
var
|
||||
nextMonth, nextYear: Integer;
|
||||
nextFile: TAuraDataFile;
|
||||
begin
|
||||
if not IsValid then
|
||||
exit(Default(TAuraDataFile));
|
||||
|
||||
nextMonth := FMonth + 1;
|
||||
nextYear := FYear;
|
||||
if (nextMonth > 12) then
|
||||
begin
|
||||
nextMonth := 1;
|
||||
Inc(nextYear);
|
||||
end;
|
||||
|
||||
// Probe for zipped file first
|
||||
nextFile := TAuraDataFile.Create(FPath, FSymbol, '.tab_zip', nextYear, nextMonth);
|
||||
if TFile.Exists(nextFile.GetFullFileName) then
|
||||
exit(nextFile);
|
||||
|
||||
// Probe for uncompressed file
|
||||
nextFile := TAuraDataFile.Create(FPath, FSymbol, '.tab', nextYear, nextMonth);
|
||||
if TFile.Exists(nextFile.GetFullFileName) then
|
||||
exit(nextFile);
|
||||
|
||||
// No next file found
|
||||
Result := Default(TAuraDataFile);
|
||||
end;
|
||||
|
||||
{ TAuraDataServer<T> }
|
||||
|
||||
constructor TAuraDataServer<T>.Create(const APath: String);
|
||||
@@ -195,8 +188,13 @@ begin
|
||||
FPath := APath;
|
||||
|
||||
FCachedFiles :=
|
||||
TDataFileCache<TArray<TArray<TDataPoint<T>>>>
|
||||
.Create(function(const Filename: String): TFuture<TArray<TArray<TDataPoint<T>>>> begin Result := DoLoad(Filename); end);
|
||||
TDataFileCache<TArray<TArray<TDataPoint<T>>>>.Create(
|
||||
function(const Filename: String): TFuture<TArray<TArray<TDataPoint<T>>>>
|
||||
begin
|
||||
// Pass the private load gate to the virtual DoLoad method.
|
||||
Result := DoLoad(Filename, FLoadGate);
|
||||
end
|
||||
);
|
||||
end;
|
||||
|
||||
destructor TAuraDataServer<T>.Destroy;
|
||||
@@ -217,92 +215,61 @@ begin
|
||||
FCachedFiles.Clear;
|
||||
end;
|
||||
|
||||
function TAuraDataServer<T>.DoLoad(const FileName: string): TFuture<TArray<TArray<TDataPoint<T>>>>;
|
||||
begin
|
||||
Result := TFuture<TArray<TArray<TDataPoint<T>>>>.Null;
|
||||
|
||||
if not TFile.Exists(FileName) then
|
||||
exit;
|
||||
|
||||
var capFileName := FileName;
|
||||
if FileName.EndsWith('_zip', True) then
|
||||
begin
|
||||
Result :=
|
||||
TFuture<TBytes>
|
||||
.Construct(FLoadGate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
|
||||
.Chain<TArray<TArray<TDataPoint<T>>>>(
|
||||
function(bytes: TBytes): TArray<TArray<TDataPoint<T>>>
|
||||
begin
|
||||
var zipMemoryStream := TBytesStream.Create(bytes);
|
||||
try
|
||||
zipMemoryStream.Position := 0;
|
||||
Result := ReadCompressedData(zipMemoryStream);
|
||||
finally
|
||||
zipMemoryStream.Free;
|
||||
end;
|
||||
end);
|
||||
end
|
||||
else
|
||||
begin
|
||||
Result :=
|
||||
TFuture<TArray<TArray<TDataPoint<T>>>>.Construct(
|
||||
FLoadGate.Enqueue,
|
||||
function: TArray<TArray<TDataPoint<T>>>
|
||||
begin
|
||||
if TFile.Exists(capFileName) then
|
||||
begin
|
||||
var stream := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
|
||||
try
|
||||
try
|
||||
Result := ReadUncompressedData(stream);
|
||||
except
|
||||
on E: EReadError do
|
||||
raise EReadError.CreateFmt('File %s: %s', [capFileName, E.Message]);
|
||||
end;
|
||||
finally
|
||||
stream.Free;
|
||||
end;
|
||||
end;
|
||||
end
|
||||
);
|
||||
end;
|
||||
end;
|
||||
|
||||
procedure TAuraDataServer<T>.UpdateSymbols;
|
||||
begin
|
||||
FSymbols :=
|
||||
TFuture<TDictionary<String, TAuraDataFile>>.Construct(
|
||||
TFuture<TDictionary<String, TArray<TAuraDataFile>>>.Construct(
|
||||
FSymbols.Done,
|
||||
function: TDictionary<String, TAuraDataFile>
|
||||
function: TDictionary<String, TArray<TAuraDataFile>>
|
||||
var
|
||||
fileNames: TArray<string>;
|
||||
currentFile: string;
|
||||
tempSymbolMap: TDictionary<String, TList<TAuraDataFile>>;
|
||||
dataFile: TAuraDataFile;
|
||||
trackedInfo: TAuraDataFile;
|
||||
symbolList: TList<TAuraDataFile>;
|
||||
begin
|
||||
Result := TDictionary<string, TAuraDataFile>.Create;
|
||||
if not TDirectory.Exists(FPath) then
|
||||
exit;
|
||||
Result := TDictionary<String, TArray<TAuraDataFile>>.Create;
|
||||
tempSymbolMap := TDictionary<String, TList<TAuraDataFile>>.Create;
|
||||
try
|
||||
if not TDirectory.Exists(FPath) then
|
||||
exit;
|
||||
|
||||
fileNames := TDirectory.GetFiles(FPath);
|
||||
for currentFile in fileNames do
|
||||
begin
|
||||
dataFile := ParseFileName(currentFile);
|
||||
if dataFile.IsValid then
|
||||
fileNames := TDirectory.GetFiles(FPath);
|
||||
for var currentFile in fileNames do
|
||||
begin
|
||||
if Result.TryGetValue(dataFile.Symbol, trackedInfo) then
|
||||
dataFile := ParseFileName(currentFile);
|
||||
if dataFile.IsValid then
|
||||
begin
|
||||
if (dataFile.Year < trackedInfo.Year)
|
||||
or ((dataFile.Year = trackedInfo.Year) and (dataFile.Month < trackedInfo.Month)) then
|
||||
if not tempSymbolMap.TryGetValue(dataFile.Symbol, symbolList) then
|
||||
begin
|
||||
Result.AddOrSetValue(dataFile.Symbol, dataFile);
|
||||
symbolList := TList<TAuraDataFile>.Create;
|
||||
tempSymbolMap.Add(dataFile.Symbol, symbolList);
|
||||
end;
|
||||
end
|
||||
else
|
||||
begin
|
||||
Result.Add(dataFile.Symbol, dataFile);
|
||||
symbolList.Add(dataFile);
|
||||
end;
|
||||
end;
|
||||
|
||||
for var kvp in tempSymbolMap do
|
||||
begin
|
||||
symbolList := kvp.Value;
|
||||
symbolList.Sort(
|
||||
TComparer<TAuraDataFile>.Construct(
|
||||
function(const Left, Right: TAuraDataFile): Integer
|
||||
begin
|
||||
if Left.Year < Right.Year then
|
||||
Result := -1
|
||||
else if Left.Year > Right.Year then
|
||||
Result := 1
|
||||
else
|
||||
Result := Left.Month - Right.Month;
|
||||
end
|
||||
)
|
||||
);
|
||||
Result.Add(kvp.Key, symbolList.ToArray);
|
||||
end;
|
||||
finally
|
||||
for var kvp in tempSymbolMap do
|
||||
kvp.Value.Free;
|
||||
tempSymbolMap.Free;
|
||||
end;
|
||||
end
|
||||
);
|
||||
@@ -314,7 +281,7 @@ function TAuraDataServer<T>.EnumerateSymbols: TFuture<TArray<String>>;
|
||||
begin
|
||||
Result :=
|
||||
FSymbols.Chain<TArray<String>>(
|
||||
function(Symbols: TDictionary<String, TAuraDataFile>): TArray<String> begin Result := Symbols.Keys.ToArray; end
|
||||
function(const Symbols: TDictionary<String, TArray<TAuraDataFile>>): TArray<String> begin Result := Symbols.Keys.ToArray; end
|
||||
);
|
||||
end;
|
||||
|
||||
@@ -323,14 +290,46 @@ begin
|
||||
var symName := Symbol;
|
||||
Result :=
|
||||
FSymbols.Chain<TAuraDataFile>(
|
||||
function(Symbols: TDictionary<String, TAuraDataFile>): TAuraDataFile
|
||||
function(const Symbols: TDictionary<String, TArray<TAuraDataFile>>): TAuraDataFile
|
||||
var
|
||||
symbolFiles: TArray<TAuraDataFile>;
|
||||
begin
|
||||
if not Symbols.TryGetValue(symName, Result) then
|
||||
if Symbols.TryGetValue(symName, symbolFiles) and (Length(symbolFiles) > 0) then
|
||||
Result := symbolFiles[0]
|
||||
else
|
||||
Result := Default(TAuraDataFile);
|
||||
end
|
||||
);
|
||||
end;
|
||||
|
||||
function TAuraDataServer<T>.GetNextFile(const Curr: TAuraDataFile): TAuraDataFile;
|
||||
var
|
||||
allSymbolFiles: TArray<TAuraDataFile>;
|
||||
foundIndex: Integer;
|
||||
begin
|
||||
Result := Default(TAuraDataFile);
|
||||
if not Curr.IsValid or not FSymbols.Done.IsSet then
|
||||
exit;
|
||||
|
||||
if not FSymbols.Value.TryGetValue(Curr.Symbol, allSymbolFiles) then
|
||||
exit;
|
||||
|
||||
foundIndex := -1;
|
||||
for var i := 0 to High(allSymbolFiles) do
|
||||
begin
|
||||
if allSymbolFiles[i].GetBaseFileName() = Curr.GetBaseFileName() then
|
||||
begin
|
||||
foundIndex := i;
|
||||
break;
|
||||
end;
|
||||
end;
|
||||
|
||||
if (foundIndex <> -1) and (foundIndex < High(allSymbolFiles)) then
|
||||
begin
|
||||
Result := allSymbolFiles[foundIndex + 1];
|
||||
end;
|
||||
end;
|
||||
|
||||
function TAuraDataServer<T>.GetPath: String;
|
||||
begin
|
||||
Result := FPath;
|
||||
@@ -357,7 +356,9 @@ end;
|
||||
function TAuraDataServer<T>.LoadDataFile(const DataFile: TAuraDataFile): TFuture<TArray<TArray<TDataPoint<T>>>>;
|
||||
begin
|
||||
if DataFile.IsValid then
|
||||
Result := FCachedFiles.GetOrAdd(DataFile.GetFullFileName);
|
||||
Result := FCachedFiles.GetOrAdd(DataFile.GetFullFileName)
|
||||
else
|
||||
Result := TFuture<TArray<TArray<TDataPoint<T>>>>.Construct(TArray<TArray<TDataPoint<T>>>(nil));
|
||||
end;
|
||||
|
||||
function TAuraDataServer<T>.ProcessChunks(
|
||||
@@ -397,7 +398,7 @@ begin
|
||||
exit(DataFile.Done);
|
||||
|
||||
// Read ahead the next file, while processing the current one
|
||||
var nextFileInfo := FileInfo.GetNextFile;
|
||||
var nextFileInfo := GetNextFile(FileInfo);
|
||||
var nextFile := LoadDataFile(nextFileInfo);
|
||||
|
||||
var cTerminated := Terminated;
|
||||
@@ -416,90 +417,52 @@ begin
|
||||
);
|
||||
end;
|
||||
|
||||
function TAuraTABFileServer.LoadDataSeries(const InitialFile: TAuraDataFile): TFuture<TArray<TDataPoint<TAuraAskBidFileItem>>>;
|
||||
var
|
||||
loadedState: TState;
|
||||
loadedFiles: TArray<TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>>;
|
||||
liveData: TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>;
|
||||
tabFiles: TList<TAuraDataFile>;
|
||||
liveFile: TAuraDataFile;
|
||||
currentFileInfo: TAuraDataFile;
|
||||
{ TAuraTABFileServer }
|
||||
|
||||
function TAuraTABFileServer.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>;
|
||||
begin
|
||||
tabFiles := TList<TAuraDataFile>.Create;
|
||||
try
|
||||
currentFileInfo := InitialFile;
|
||||
while currentFileInfo.IsValid do
|
||||
begin
|
||||
tabFiles.Add(currentFileInfo);
|
||||
currentFileInfo := currentFileInfo.GetNextFile;
|
||||
end;
|
||||
Result := TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>.Null;
|
||||
|
||||
liveFile := Default(TAuraDataFile);
|
||||
if tabFiles.Count > 0 then
|
||||
begin
|
||||
var lastTabFile := tabFiles.Last;
|
||||
var liveFileBaseName := Format('%s_%d_%02d.tab-live', [lastTabFile.Symbol, lastTabFile.Year, lastTabFile.Month]);
|
||||
var potentialLivePath := TPath.Combine(lastTabFile.Path, liveFileBaseName);
|
||||
if TFile.Exists(potentialLivePath) then
|
||||
liveFile := ParseFileName(potentialLivePath);
|
||||
end;
|
||||
if not TFile.Exists(FileName) then
|
||||
exit;
|
||||
|
||||
var loadedFileList := TList<TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>>.Create;
|
||||
var loadStates := TList<TState>.Create;
|
||||
try
|
||||
for var fileInfo in tabFiles do
|
||||
begin
|
||||
var data := LoadDataFile(fileInfo);
|
||||
loadedFileList.Add(data);
|
||||
loadStates.Add(data.Done);
|
||||
end;
|
||||
|
||||
liveData := TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>.Null;
|
||||
if liveFile.IsValid then
|
||||
begin
|
||||
liveData := LoadDataFile(liveFile);
|
||||
loadedFileList.Add(liveData);
|
||||
loadStates.Add(liveData.Done);
|
||||
end;
|
||||
|
||||
loadedState := TState.All(loadStates.ToArray);
|
||||
loadedFiles := loadedFileList.ToArray;
|
||||
finally
|
||||
loadStates.Free;
|
||||
loadedFileList.Free;
|
||||
end;
|
||||
finally
|
||||
tabFiles.Free;
|
||||
var capFileName := FileName;
|
||||
if FileName.EndsWith('_zip', True) then
|
||||
begin
|
||||
Result :=
|
||||
TFuture<TBytes>
|
||||
.Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
|
||||
.Chain<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>(
|
||||
function(bytes: TBytes): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>
|
||||
begin
|
||||
var compMemoryStream := TBytesStream.Create(bytes);
|
||||
try
|
||||
compMemoryStream.Position := 0;
|
||||
Result := ReadCompressedData(compMemoryStream);
|
||||
finally
|
||||
compMemoryStream.Free;
|
||||
end;
|
||||
end);
|
||||
end
|
||||
else
|
||||
begin
|
||||
Result :=
|
||||
TFuture<TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>>.Construct(
|
||||
Gate.Enqueue,
|
||||
function: TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>
|
||||
begin
|
||||
if TFile.Exists(capFileName) then
|
||||
begin
|
||||
var stream := TFileStream.Create(capFileName, fmOpenRead or fmShareDenyWrite);
|
||||
try
|
||||
Result := ReadUncompressedData(stream);
|
||||
finally
|
||||
stream.Free;
|
||||
end;
|
||||
end;
|
||||
end
|
||||
);
|
||||
end;
|
||||
|
||||
Result :=
|
||||
TFuture<TArray<TDataPoint<TAuraAskBidFileItem>>>.Construct(
|
||||
loadedState,
|
||||
function: TArray<TDataPoint<TAuraAskBidFileItem>>
|
||||
begin
|
||||
var tickList := TList<TDataPoint<TAuraAskBidFileItem>>.Create;
|
||||
try
|
||||
var overallLastTabTickTime: TDateTime := 0;
|
||||
var cnt := 0;
|
||||
for var P in loadedFiles do
|
||||
for var chunk in P.Value do
|
||||
inc(cnt, Length(chunk));
|
||||
tickList.Capacity := cnt;
|
||||
|
||||
for var P in loadedFiles do
|
||||
for var chunk in P.Value do
|
||||
for var iterTickData in chunk do
|
||||
if overallLastTabTickTime < iterTickData.Time then
|
||||
begin
|
||||
tickList.Add(iterTickData);
|
||||
overallLastTabTickTime := iterTickData.Time;
|
||||
end;
|
||||
Result := tickList.ToArray;
|
||||
finally
|
||||
tickList.Free;
|
||||
end;
|
||||
end
|
||||
);
|
||||
end;
|
||||
|
||||
function TAuraTABFileServer.ParseFileName(const FileName: string): TAuraDataFile;
|
||||
@@ -546,7 +509,7 @@ begin
|
||||
if symbol = '' then
|
||||
exit;
|
||||
|
||||
Result := TAuraDataFile.Create(path, symbol, fileExt, year, month);
|
||||
Result := TAuraDataFile.Create(path, symbol, fileExt, year, month, baseName);
|
||||
end;
|
||||
|
||||
class function TAuraTABFileServer.ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TAuraAskBidFileItem>>>;
|
||||
@@ -628,7 +591,193 @@ begin
|
||||
if bytesRead <> SizeOf(TFileRecord) then
|
||||
raise EReadError.CreateFmt('Read error. Expected %d bytes, read %d.', [SizeOf(TFileRecord), bytesRead]);
|
||||
|
||||
Result[chunkIndex][indexInChunk].Create(rec.TimeStamp, rec.Data);
|
||||
// Corrected syntax: Assign the newly created record instance.
|
||||
Result[chunkIndex][indexInChunk] := TDataPoint<TAuraAskBidFileItem>.Create(rec.TimeStamp, rec.Data);
|
||||
Inc(indexInChunk);
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
{ TAuraM1FileServer }
|
||||
|
||||
function TAuraM1FileServer.DoLoad(const FileName: string; const Gate: TLatch): TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>;
|
||||
begin
|
||||
Result := TFuture<TArray<TArray<TDataPoint<TOhlcItem>>>>.Null;
|
||||
|
||||
if not TFile.Exists(FileName) then
|
||||
exit;
|
||||
|
||||
var capFileName := FileName;
|
||||
// M1 files are always compressed with the .m1 extension
|
||||
if FileName.EndsWith('.m1', True) then
|
||||
begin
|
||||
Result :=
|
||||
TFuture<TBytes>
|
||||
.Construct(Gate.Enqueue, function: TBytes begin Result := TFile.ReadAllBytes(capFileName); end)
|
||||
.Chain<TArray<TArray<TDataPoint<TOhlcItem>>>>(
|
||||
function(bytes: TBytes): TArray<TArray<TDataPoint<TOhlcItem>>>
|
||||
begin
|
||||
var compMemoryStream := TBytesStream.Create(bytes);
|
||||
try
|
||||
compMemoryStream.Position := 0;
|
||||
Result := ReadCompressedData(compMemoryStream);
|
||||
finally
|
||||
compMemoryStream.Free;
|
||||
end;
|
||||
end);
|
||||
end;
|
||||
end;
|
||||
|
||||
function TAuraM1FileServer.ParseFileName(const FileName: string): TAuraDataFile;
|
||||
var
|
||||
path, fileNameNoPath, baseName, ext, symbol: string;
|
||||
year, month: Integer;
|
||||
parts, dateParts: TArray<string>;
|
||||
begin
|
||||
Result := Default(TAuraDataFile);
|
||||
path := TPath.GetDirectoryName(FileName);
|
||||
fileNameNoPath := TPath.GetFileName(FileName);
|
||||
|
||||
// M1 files are compressed and have a .m1 extension
|
||||
if not fileNameNoPath.EndsWith('.m1', True) then
|
||||
exit;
|
||||
|
||||
ext := TPath.GetExtension(fileNameNoPath);
|
||||
baseName := TPath.GetFileNameWithoutExtension(fileNameNoPath);
|
||||
|
||||
// Expected format: SYMBOL_YYYY-MM_YYYY-MM
|
||||
parts := baseName.Split(['_']);
|
||||
if Length(parts) < 3 then
|
||||
exit; // Not enough parts for SYMBOL_START_END
|
||||
|
||||
// The symbol can contain underscores, so we take everything before the last two parts.
|
||||
var startDatePart := parts[High(parts) - 1];
|
||||
|
||||
dateParts := startDatePart.Split(['-']);
|
||||
if Length(dateParts) <> 2 then
|
||||
exit; // Start date is not YYYY-MM
|
||||
|
||||
if not TryStrToInt(dateParts[0], year) then
|
||||
exit;
|
||||
|
||||
if not TryStrToInt(dateParts[1], month) then
|
||||
exit;
|
||||
|
||||
if (month < 1) or (month > 12) or (year <= 0) then
|
||||
exit;
|
||||
|
||||
symbol := string.Join('_', Copy(parts, 0, Length(parts) - 2));
|
||||
if symbol = '' then
|
||||
exit;
|
||||
|
||||
// The TAuraDataFile only represents the start date of the file batch.
|
||||
Result := TAuraDataFile.Create(path, symbol, ext, year, month, baseName);
|
||||
end;
|
||||
|
||||
class function TAuraM1FileServer.ReadCompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
|
||||
var
|
||||
decompressionStream: TStream;
|
||||
localHeader: TZipHeader;
|
||||
entryIndex, i: Integer;
|
||||
zipFileInstance: TZipFile;
|
||||
begin
|
||||
Result := nil;
|
||||
decompressionStream := nil;
|
||||
zipFileInstance := nil;
|
||||
|
||||
// M1 files are now zip archives containing the actual data file.
|
||||
if not Assigned(InputStream) or (InputStream.Size = 0) then
|
||||
exit;
|
||||
|
||||
try
|
||||
InputStream.Position := 0;
|
||||
zipFileInstance := TZipFile.Create;
|
||||
zipFileInstance.Open(InputStream, TZipMode.zmRead);
|
||||
|
||||
if zipFileInstance.FileCount = 0 then
|
||||
exit;
|
||||
|
||||
// Find the data file entry within the archive.
|
||||
// The C# code creates an entry with the same name as the base filename, ending in .m1.
|
||||
entryIndex := -1;
|
||||
for i := 0 to zipFileInstance.FileCount - 1 do
|
||||
begin
|
||||
if zipFileInstance.FileNames[i].EndsWith('.m1', True) then
|
||||
begin
|
||||
entryIndex := i;
|
||||
break;
|
||||
end;
|
||||
end;
|
||||
|
||||
// Fallback to the first entry if no specific .m1 file is found inside.
|
||||
if entryIndex = -1 then
|
||||
entryIndex := 0;
|
||||
|
||||
zipFileInstance.Read(entryIndex, decompressionStream, localHeader);
|
||||
if not Assigned(decompressionStream) then
|
||||
exit;
|
||||
|
||||
// The decompressed stream now contains the raw binary data.
|
||||
Result := ReadUncompressedData(decompressionStream);
|
||||
finally
|
||||
decompressionStream.Free;
|
||||
zipFileInstance.Free;
|
||||
end;
|
||||
end;
|
||||
|
||||
class function TAuraM1FileServer.ReadUncompressedData(const InputStream: TStream): TArray<TArray<TDataPoint<TOhlcItem>>>;
|
||||
type
|
||||
TFileRecord = TAuraM1FileItem;
|
||||
var
|
||||
fileSize: Int64;
|
||||
totalRecordCount, bytesRead, chunkIndex, indexInChunk, recordIndex: Integer;
|
||||
rec: TFileRecord;
|
||||
ohlc: TOhlcItem;
|
||||
factor: Double;
|
||||
begin
|
||||
Result := nil;
|
||||
InputStream.Position := 0;
|
||||
fileSize := InputStream.Size;
|
||||
if (fileSize = 0) or ((fileSize mod SizeOf(TFileRecord)) <> 0) then
|
||||
exit;
|
||||
|
||||
totalRecordCount := fileSize div SizeOf(TFileRecord);
|
||||
if totalRecordCount > 0 then
|
||||
begin
|
||||
var numChunks := (totalRecordCount + ChunkSize - 1) div ChunkSize;
|
||||
SetLength(Result, numChunks);
|
||||
chunkIndex := 0;
|
||||
indexInChunk := 0;
|
||||
SetLength(Result[chunkIndex], Min(ChunkSize, totalRecordCount));
|
||||
|
||||
for recordIndex := 0 to totalRecordCount - 1 do
|
||||
begin
|
||||
if indexInChunk >= ChunkSize then
|
||||
begin
|
||||
Inc(chunkIndex);
|
||||
indexInChunk := 0;
|
||||
var remainingRecords := totalRecordCount - (chunkIndex * ChunkSize);
|
||||
SetLength(Result[chunkIndex], Min(ChunkSize, remainingRecords));
|
||||
end;
|
||||
|
||||
bytesRead := InputStream.Read(rec, SizeOf(TFileRecord));
|
||||
if bytesRead <> SizeOf(TFileRecord) then
|
||||
raise EReadError.CreateFmt('Read error. Expected %d bytes, read %d.', [SizeOf(TFileRecord), bytesRead]);
|
||||
|
||||
// Convert from M1 file record to a standard OHLC item
|
||||
if rec.Digits > 0 then
|
||||
factor := Power(10, rec.Digits)
|
||||
else
|
||||
factor := 1.0;
|
||||
|
||||
ohlc.Open := rec.Open / factor;
|
||||
ohlc.High := rec.High / factor;
|
||||
ohlc.Low := rec.Low / factor;
|
||||
ohlc.Close := rec.Close / factor;
|
||||
ohlc.Volume := rec.TickVolume;
|
||||
|
||||
// Corrected syntax: Assign the newly created record instance.
|
||||
Result[chunkIndex][indexInChunk] := TDataPoint<TOhlcItem>.Create(rec.OADateTime, ohlc);
|
||||
Inc(indexInChunk);
|
||||
end;
|
||||
end;
|
||||
|
||||
Reference in New Issue
Block a user