Brummel/MycLib
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Streamlining DataServer

Browse Source
This commit is contained in:
Michael Schimmel
2025-06-07 10:12:41 +02:00
parent 031b99acc8
commit 2cdae3d3f6
5 changed files with 148 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Src/Myc.Futures.pas
+10 -1
View File
@@ -22,6 +22,8 @@ type
property Done: TState read GetDone; property Done: TState read GetDone;
end; end;
TFuncConst<T, TResult> = reference to function(const Arg1: T): TResult;
{$REGION 'private'} {$REGION 'private'}
strict private strict private
class var class var
@@ -47,7 +49,8 @@ type
class property Null: IFuture read FNull; class property Null: IFuture read FNull;
function Chain<S>(const Proc: TFunc<T, S>): TFuture<S>; function Chain<S>(const Proc: TFunc<T, S>): TFuture<S>; overload;
function Chain<S>(const Proc: TFuncConst<T, S>): TFuture<S>; overload;
function WaitFor: T; function WaitFor: T;
property Done: TState read GetDone; property Done: TState read GetDone;
@@ -84,6 +87,12 @@ begin
Result := TFuture<S>.Construct(FFuture.Done, function: S begin Result := Proc(Cap.Value); end); Result := TFuture<S>.Construct(FFuture.Done, function: S begin Result := Proc(Cap.Value); end);
end; end;
function TFuture<T>.Chain<S>(const Proc: TFuncConst<T, S>): TFuture<S>;
begin
var Cap := FFuture;
Result := TFuture<S>.Construct(FFuture.Done, function: S begin Result := Proc(Cap.Value); end);
end;
class function TFuture<T>.Construct(const Proc: TFunc<T>): TFuture<T>; class function TFuture<T>.Construct(const Proc: TFunc<T>): TFuture<T>;
begin begin
Result := TMycGateFuncFuture<T>.Create(TaskManager, nil, Proc); Result := TMycGateFuncFuture<T>.Create(TaskManager, nil, Proc);
Src/Myc.Test.Trade.DataServer.pas
+50 -58
View File
@@ -14,19 +14,20 @@ uses
Myc.Trade.DataServer; Myc.Trade.DataServer;
type type
// Define ITickData as an alias for the specific IDataPoint interface.
ITickData = IDataPoint<ITick>;
[TestFixture] [TestFixture]
[IgnoreMemoryLeaks(true)]
TTest_TABFileServer_Equivalence = class(TObject) TTest_TABFileServer_Equivalence = class(TObject)
private private
FServer: IDataServer<ITickData>; FServer: IDataServer<TDataPoint<TAskBidItem>>;
FExpectedData: TArray<TAskBid.TTick>;
public public
[SetupFixture]
procedure SetupFixture;
[Setup] [Setup]
procedure Setup; procedure Setup;
[TearDown] [TearDown]
procedure TearDown; procedure TearDown;
[TearDownFixture]
procedure TearDownFixture;
[Test] [Test]
procedure Test_ServerReturnsSameDataAs_LoadDataSeries; procedure Test_ServerReturnsSameDataAs_LoadDataSeries;
@@ -39,79 +40,70 @@ uses
const const
// The reference data file for this test. // The reference data file for this test.
C_TEST_FILENAME = '\\COFFEE\TickData\Pepperstone\AUDNZD_2024_05.tab_zip'; C_TEST_FILENAME = '\\COFFEE\TickData\Pepperstone\GER40_2025_03.tab_zip';
// The chunk size for fetching data from the server. // The chunk size for fetching data from the server.
C_MAX_FETCH = 20; C_MAX_FETCH = 200;
{ TTest_TABFileServer_Equivalence } { TTest_TABFileServer_Equivalence }
procedure TTest_TABFileServer_Equivalence.SetupFixture;
begin
// IsMemoryLow := function: Boolean begin exit(false); end;
// Load the ground truth data using the original LoadDataSeries function.
// SetupTaskManagerMock;
TAskBid.ClearCache;
end;
procedure TTest_TABFileServer_Equivalence.TearDownFixture;
begin
IsMemoryLow := nil;
end;
procedure TTest_TABFileServer_Equivalence.Setup; procedure TTest_TABFileServer_Equivalence.Setup;
begin begin
// SetupTaskManagerMock; FServer := TTABFileServer.Create(C_TEST_FILENAME);
// IsMemoryLow := function: Boolean begin exit(false); end;
// 1. Load the ground truth data using the original LoadDataSeries function.
FExpectedData := TAskBid.LoadDataSeries(C_TEST_FILENAME).WaitFor;
// 2. Set up the filename and the TABFileServer instance.
FServer := TTABFileServer.Create(C_TEST_FILENAME, C_MAX_FETCH);
end; end;
procedure TTest_TABFileServer_Equivalence.TearDown; procedure TTest_TABFileServer_Equivalence.TearDown;
begin begin
FServer := nil; FServer := nil;
SetLength(FExpectedData, 0); TAskBid.ClearCache;
IsMemoryLow := nil;
end; end;
procedure TTest_TABFileServer_Equivalence.Test_ServerReturnsSameDataAs_LoadDataSeries; procedure TTest_TABFileServer_Equivalence.Test_ServerReturnsSameDataAs_LoadDataSeries;
var var
actualDataList: TList<ITickData>; chunk: array[0..C_MAX_FETCH-1] of TDataPoint<TAskBidItem>;
chunk: TArray<ITickData>; Dst: TArray<TDataPoint<TAskBidItem>>;
newChunk: TLazy<TArray<ITickData>>; i: Int64;
begin begin
actualDataList := TList<ITickData>.Create; var ExpectedData := TAskBid.LoadDataSeries(C_TEST_FILENAME).WaitFor;
try SetLength( Dst, Length( ExpectedData ) );
newChunk := TLazy<TArray<ITickData>>.Construct(FServer.Data);
// Loop until the server stops providing new data chunks. var cnt: Int64 := 0;
var cnt: Int64 := 0; var n: Integer;
while not FServer.IsLiveData.Value do while not FServer.IsLiveData.Value and (cnt < Length(Dst)) do
begin begin
FServer.Fetch; n := FServer.GetChunk(chunk);
if n > 0 then
Move( chunk[0], dst[cnt], n * sizeof( TDataPoint<TAskBidItem> ) );
inc( cnt, n );
end;
// Get the data chunk produced by the last Fetch call. // --- Assertions ---
if newChunk.Pop(chunk) then
begin
for var i := 0 to High(chunk) do
begin
actualDataList.Add(chunk[i]);
Assert.AreEqual(chunk[i].Idx, cnt, 'Index mismatch');
inc(cnt);
end;
end;
end;
// --- Assertions --- // 1. Verify that the total number of records matches.
Assert.AreEqual(Length(ExpectedData), cnt, 'Data record count mismatch');
// 1. Verify that the total number of records matches. // 2. Verify that the content of each record is identical.
Assert.AreEqual(Length(FExpectedData), actualDataList.Count, 'Data record count mismatch'); for n := 0 to High(ExpectedData) div 1000 do
begin
// 2. Verify that the content of each record is identical. i := n * 1000;
for var i := 0 to High(FExpectedData) do // E2003: Removed assertion for 'Index', as it's not part of the IDataPoint interface.
begin // The index is an implementation detail of the faulty ConvertTicks and cannot be tested here.
var expectedTick := FExpectedData[i]; Assert.AreEqual(Dst[i].Idx, i, 'Index mismatch ');
var actualTick := actualDataList[i]; Assert.AreEqual(ExpectedData[i].TimeStamp, Dst[i].Time, 'Timestamp mismatch ');
Assert.AreEqual(ExpectedData[i].Data.Ask, Dst[i].Data.Ask, 'Ask price mismatch');
// E2003: Removed assertion for 'Index', as it's not part of the IDataPoint interface. Assert.AreEqual(ExpectedData[i].Data.Bid, Dst[i].Data.Bid, 'Bid price mismatch');
// The index is an implementation detail of the faulty ConvertTicks and cannot be tested here.
Assert.AreEqual(expectedTick.OADateTime, actualTick.Time, 'Timestamp mismatch ');
Assert.IsTrue(Abs(expectedTick.Data.Ask - actualTick.Data.Ask) < 0.0001, 'Ask price mismatch');
Assert.IsTrue(Abs(expectedTick.Data.Bid - actualTick.Data.Bid) < 0.0001, 'Bid price mismatch');
end;
finally
actualDataList.Free;
end; end;
end; end;
Src/Myc.Trade.DataPoint.pas
+13 -148
View File
@@ -6,163 +6,28 @@ uses
System.SysUtils; System.SysUtils;
type type
IDataPoint<T> = interface // A specific data record structure for Ask and Bid prices.
['{6A52697A-2868-42A9-982D-993542B7B360}'] TAskBidItem = packed record
function GetData: T; Ask: Single;
function GetIdx: Int64; Bid: Single;
function GetTime: TDateTime;
function GetVolume: Double;
property Time: TDateTime read GetTime;
property Volume: Double read GetVolume;
property Data: T read GetData;
property Idx: Int64 read GetIdx;
end; end;
ITick = interface TDataPoint<T> = record
['{B342223B-D299-4A1C-82F3-569F54417A66}'] Idx: Int64;
function GetAsk: Double; Time: TDateTime;
function GetBid: Double; Data: T;
constructor Create(AIdx: Int64; ATime: TDateTime; const AData: T);
property Ask: Double read GetAsk;
property Bid: Double read GetBid;
end;
ITickData = IDataPoint<ITick>;
IOHLC = interface
['{05374E3C-731B-44FA-A23E-051F1461B78D}']
function GetOpen: Double;
function GetHigh: Double;
function GetLow: Double;
function GetClose: Double;
property Open: Double read GetOpen;
property High: Double read GetHigh;
property Low: Double read GetLow;
property Close: Double read GetClose;
end;
IOHLCData = IDataPoint<IOHLC>;
// Abstract base class for all data points.
TDataPoint<T> = class abstract(TInterfacedObject, IDataPoint<T>)
protected
fTime: TDateTime;
fVolume: Double;
FData: T;
FIdx: Int64;
function GetTime: TDateTime;
function GetVolume: Double;
function GetData: T;
function GetIdx: Int64;
public
constructor Create(Idx: Int64; Time: TDateTime; Volume: Double; const AData: T);
end;
TTick = class(TInterfacedObject, ITick)
private
fAsk: Double;
fBid: Double;
function GetAsk: Double;
function GetBid: Double;
public
constructor Create(Ask: Double; Bid: Double);
end;
TOHLC = class(TInterfacedObject, IOHLC)
private
fOpen: Double;
fHigh: Double;
fLow: Double;
fClose: Double;
function GetClose: Double;
function GetHigh: Double;
function GetLow: Double;
function GetOpen: Double;
public
constructor Create(Open, High, Low, Close: Double);
end; end;
implementation implementation
{ TDataPoint } { TDataPoint }
constructor TDataPoint<T>.Create(Idx: Int64; Time: TDateTime; Volume: Double; const AData: T); constructor TDataPoint<T>.Create(AIdx: Int64; ATime: TDateTime; const AData: T);
begin begin
inherited Create; Idx := AIdx;
FIdx := Idx; Time := ATime;
fTime := Time; Data := AData;
fVolume := Volume;
FData := AData;
end;
function TDataPoint<T>.GetData: T;
begin
Result := FData;
end;
function TDataPoint<T>.GetIdx: Int64;
begin
Result := FIdx;
end;
function TDataPoint<T>.GetTime: TDateTime;
begin
result := fTime;
end;
function TDataPoint<T>.GetVolume: Double;
begin
result := fVolume;
end;
{ TTick }
constructor TTick.Create(Ask: Double; Bid: Double);
begin
inherited Create;
fAsk := Ask;
fBid := Bid;
end;
function TTick.GetAsk: Double;
begin
result := fAsk;
end;
function TTick.GetBid: Double;
begin
result := fBid;
end;
{ TOHLC }
constructor TOHLC.Create(Open, High, Low, Close: Double);
begin
inherited Create;
fOpen := Open;
fHigh := High;
fLow := Low;
fClose := Close;
end;
function TOHLC.GetClose: Double;
begin
result := fClose;
end;
function TOHLC.GetHigh: Double;
begin
result := fHigh;
end;
function TOHLC.GetLow: Double;
begin
result := fLow;
end;
function TOHLC.GetOpen: Double;
begin
result := fOpen;
end; end;
end. end.
Src/Myc.Trade.DataSeries.pas
+26 -23
View File
@@ -45,7 +45,8 @@ uses
System.Generics.Defaults, System.Generics.Defaults,
System.IOUtils, System.IOUtils,
Myc.Futures, Myc.Futures,
Myc.Signals; Myc.Signals,
Myc.Trade.DataPoint;
type type
// Generic class for loading and managing sequential time-series data. // Generic class for loading and managing sequential time-series data.
@@ -54,13 +55,13 @@ type
type type
// Represents a single data point with a timestamp and generic data. // Represents a single data point with a timestamp and generic data.
TDataPoint = packed record TDataPoint = packed record
OADateTime: TDateTime; TimeStamp: TDateTime;
Data: T; Data: T;
end; end;
strict private strict private
type type
TCachedFile = record TCachedFile = class
Name: String; Name: String;
Age: TDateTime; Age: TDateTime;
LastUsed: TDateTime; LastUsed: TDateTime;
@@ -68,6 +69,7 @@ type
end; end;
class var class var
FCachedFiles: TDictionary<String, TCachedFile>; FCachedFiles: TDictionary<String, TCachedFile>;
FLoadGate: TLatch;
class constructor CreateClass; class constructor CreateClass;
class destructor DestroyClass; class destructor DestroyClass;
@@ -76,16 +78,12 @@ type
class function ReadCompressedData(const InputStream: TStream): TArray<TDataPoint>; static; class function ReadCompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
class function ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint>; static; class function ReadUncompressedData(const InputStream: TStream): TArray<TDataPoint>; static;
public public
class procedure ClearCache;
// Asynchronously loads a single data file with caching support. // Asynchronously loads a single data file with caching support.
class function LoadDataFile(const LoadGate: TState; const FileName: string): TFuture<TArray<TDataPoint>>; static; class function LoadDataFile(const FileName: string): TFuture<TArray<TDataPoint>>; static;
// Asynchronously loads a complete chronological series of data files. // Asynchronously loads a complete chronological series of data files.
class function LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint>>; static; class function LoadDataSeries(const FileName: string): TFuture<TArray<TDataPoint>>; static;
end;
// A specific data record structure for Ask and Bid prices.
TAskBidItem = packed record
Ask: Single;
Bid: Single;
end; end;
// A specialized TDataSeries for handling Ask/Bid tick data. // A specialized TDataSeries for handling Ask/Bid tick data.
@@ -315,7 +313,7 @@ end;
class destructor TDataSeries<T>.CreateClass; class destructor TDataSeries<T>.CreateClass;
begin begin
FCachedFiles := TDictionary<String, TCachedFile>.Create; FCachedFiles := TObjectDictionary<String, TCachedFile>.Create([doOwnsValues]);
end; end;
class destructor TDataSeries<T>.DestroyClass; class destructor TDataSeries<T>.DestroyClass;
@@ -323,9 +321,14 @@ begin
FCachedFiles.Free; FCachedFiles.Free;
end; end;
class procedure TDataSeries<T>.ClearCache;
begin
FCachedFiles.Clear;
end;
// Asynchronously reads tick data from a single specified file. // Asynchronously reads tick data from a single specified file.
// Returns a TFuture that will provide an array of TAskBidSeries. // Returns a TFuture that will provide an array of TAskBidSeries.
class function TDataSeries<T>.LoadDataFile(const LoadGate: TState; const FileName: string): TFuture<TArray<TDataPoint>>; class function TDataSeries<T>.LoadDataFile(const FileName: string): TFuture<TArray<TDataPoint>>;
var var
parsedPath, parsedSymbol: string; parsedPath, parsedSymbol: string;
parsedYear, parsedMonth: Integer; parsedYear, parsedMonth: Integer;
@@ -351,24 +354,24 @@ begin
begin begin
if not IsMemoryLow then if not IsMemoryLow then
break; break;
FCachedFiles.Remove(fL[i].Name); if fL[i].Name <> FileName then
FCachedFiles.Remove(fL[i].Name);
end; end;
end; end;
var cachedFile: TCachedFile; var cachedFile: TCachedFile;
if FCachedFiles.TryGetValue(Filename, cachedFile) then if FCachedFiles.TryGetValue(Filename, cachedFile) then
begin begin
FCachedFiles.Remove(FileName);
var age: TDateTime; var age: TDateTime;
if FileAge(Filename, age, true) then if FileAge(Filename, age, true) then
begin begin
if age = cachedFile.Age then if age = cachedFile.Age then
begin begin
cachedFile.LastUsed := Now; cachedFile.LastUsed := Now;
FCachedFiles.Add(Filename, cachedFile);
exit(cachedFile.Data); exit(cachedFile.Data);
end; end
else
FCachedFiles.Remove(FileName);
end; end;
end; end;
end; end;
@@ -379,7 +382,7 @@ begin
Result := Result :=
TFuture<TBytes> TFuture<TBytes>
.Construct( .Construct(
LoadGate, TLatch.Enqueue(FLoadGate),
function: TBytes function: TBytes
begin begin
Result := TFile.ReadAllBytes(capFileName); // Read all bytes of the .zip file Result := TFile.ReadAllBytes(capFileName); // Read all bytes of the .zip file
@@ -401,7 +404,7 @@ begin
var capFileName := FileName; var capFileName := FileName;
Result := Result :=
TFuture<TArray<TDataPoint>>.Construct( TFuture<TArray<TDataPoint>>.Construct(
LoadGate, TLatch.Enqueue(FLoadGate),
function: TArray<TDataPoint> function: TArray<TDataPoint>
begin begin
if TFile.Exists(capFileName) then if TFile.Exists(capFileName) then
@@ -426,7 +429,7 @@ begin
if Assigned(IsMemoryLow) then if Assigned(IsMemoryLow) then
begin begin
var cachedFile: TCachedFile; var cachedFile := TCachedFile.Create;
cachedFile.Name := Filename; cachedFile.Name := Filename;
if FileAge(Filename, cachedFile.Age, true) then if FileAge(Filename, cachedFile.Age, true) then
begin begin
@@ -486,7 +489,7 @@ begin
// Create load tasks for the historical .tab files. // Create load tasks for the historical .tab files.
for currentFile in tabFiles do for currentFile in tabFiles do
begin begin
var data := LoadDataFile(TLatch.Enqueue(LoadGate), currentFile); var data := LoadDataFile(currentFile);
loadedFileList.Add(data); loadedFileList.Add(data);
loadStates.Add(data.Done); loadStates.Add(data.Done);
end; end;
@@ -495,7 +498,7 @@ begin
liveData := TFuture<TArray<TDataPoint>>.Null; liveData := TFuture<TArray<TDataPoint>>.Null;
if liveFilePath <> '' then if liveFilePath <> '' then
begin begin
liveData := LoadDataFile(TLatch.Enqueue(LoadGate), liveFilePath); liveData := LoadDataFile(liveFilePath);
loadedFileList.Add(liveData); loadedFileList.Add(liveData);
loadStates.Add(liveData.Done); loadStates.Add(liveData.Done);
end; end;
@@ -529,10 +532,10 @@ begin
for var P in loadedFiles do for var P in loadedFiles do
begin begin
for var iterTickData in P.Value do for var iterTickData in P.Value do
if overallLastTabTickTime < iterTickData.OADateTime then if overallLastTabTickTime < iterTickData.TimeStamp then
begin begin
tickList.Add(iterTickData); tickList.Add(iterTickData);
overallLastTabTickTime := iterTickData.OADateTime; overallLastTabTickTime := iterTickData.TimeStamp;
end; end;
end; end;
Result := tickList.ToArray; Result := tickList.ToArray;
Src/Myc.Trade.DataServer.pas
+49 -90
View File
@@ -12,24 +12,17 @@ uses
Myc.Futures; // Added for TFuture Myc.Futures; // Added for TFuture
type type
IDataProvider = interface IDataServer<T> = interface
['{B2A9996C-724B-4416-A882-595B7E58066D}']
procedure Fetch;
end;
IDataServer<T> = interface(IDataProvider)
['{A6E246A2-E84E-49AB-A63E-333E561E488C}'] ['{A6E246A2-E84E-49AB-A63E-333E561E488C}']
function GetData: TMutable<TArray<T>>;
function GetIsLiveData: TMutable<Boolean>; function GetIsLiveData: TMutable<Boolean>;
property Data: TMutable<TArray<T>> read GetData; function GetChunk(var Data: array of T): Integer;
property IsLiveData: TMutable<Boolean> read GetIsLiveData; property IsLiveData: TMutable<Boolean> read GetIsLiveData;
end; end;
TDataServer<T> = class(TInterfacedObject, IDataServer<T>, IDataProvider) TDataServer<T> = class(TInterfacedObject, IDataServer<T>)
protected protected
function GetData: TMutable<TArray<T>>; virtual; abstract;
function GetIsLiveData: TMutable<Boolean>; virtual; abstract; function GetIsLiveData: TMutable<Boolean>; virtual; abstract;
procedure Fetch; virtual; abstract; function GetChunk(var Data: array of T): Integer; virtual; abstract;
end; end;
IDataServerNode<T> = interface IDataServerNode<T> = interface
@@ -43,13 +36,10 @@ type
function CreateDataServer: IDataServer<T>; virtual; abstract; function CreateDataServer: IDataServer<T>; virtual; abstract;
end; end;
TTABFileServer = class(TDataServer<ITickData>) TTABFileServer = class(TDataServer<TDataPoint<TAskBidItem>>)
private private
FFilename: String; FFilename: String;
// The maximum number of data points to be produced per Fetch call.
FMaxFetch: Integer;
FHasNewData: TEvent;
FNeedNewChunk: Boolean;
FIsLiveData: TMutable<Boolean>.IWriteable; FIsLiveData: TMutable<Boolean>.IWriteable;
// Internal state // Internal state
@@ -57,21 +47,16 @@ type
FCurrentData: TFuture<TArray<TAskBid.TTick>>; FCurrentData: TFuture<TArray<TAskBid.TTick>>;
FNextFileName: string; FNextFileName: string;
FNextData: TFuture<TArray<TAskBid.TTick>>; FNextData: TFuture<TArray<TAskBid.TTick>>;
FData: TMutable<TArray<ITickData>>;
FCurrPosInFile: Int64; FCurrPosInFile: Int64;
FCurrChunk: TArray<ITickData>;
FLoadGate: TLatch;
FCurrentIdx: Int64; FCurrentIdx: Int64;
FLastTimeStamp: TDateTime; FLastTimeStamp: TDateTime;
function ConvertTicks(var Idx: Int64; const Ticks: TArray<TAskBid.TTick>): TArray<ITickData>;
function UpdateChunk: TArray<ITickData>;
protected protected
procedure Fetch; override; function GetChunk(var Data: array of TDataPoint<TAskBidItem>): Integer; override;
function GetData: TMutable<TArray<ITickData>>; override;
function GetIsLiveData: TMutable<Boolean>; override; function GetIsLiveData: TMutable<Boolean>; override;
public public
constructor Create(const AFilename: String; AMaxFetch: Integer); constructor Create(const AFilename: String);
procedure AfterConstruction; override;
end; end;
implementation implementation
@@ -81,78 +66,45 @@ uses
{ TTABFileServer<T> } { TTABFileServer<T> }
constructor TTABFileServer.Create(const AFilename: String; AMaxFetch: Integer); constructor TTABFileServer.Create(const AFilename: String);
begin begin
inherited Create; inherited Create;
FFilename := AFilename; FFilename := AFilename;
FMaxFetch := AMaxFetch;
FNextData := TFuture<TArray<TAskBid.TTick>>.Null; FNextData := TFuture<TArray<TAskBid.TTick>>.Null;
FNeedNewChunk := true;
FIsLiveData := TMutable<Boolean>.CreateWriteable(false); FIsLiveData := TMutable<Boolean>.CreateWriteable(false);
FHasNewData := TEvent.CreateEvent;
FData := TMutable<TArray<ITickData>>.Construct(FHasNewData.Signal, function: TArray<ITickData> begin Result := UpdateChunk; end);
end; end;
function TTABFileServer.ConvertTicks(var Idx: Int64; const Ticks: TArray<TAskBid.TTick>): TArray<ITickData>; procedure TTABFileServer.AfterConstruction;
begin begin
SetLength(Result, Length(Ticks)); inherited;
for var i := 0 to High(Ticks) do
begin FCurrentFileName := FFilename;
Result[i] := TDataPoint<ITick>.Create(Idx, Ticks[i].OADateTime, 1, TTick.Create(Ticks[i].Data.Ask, Ticks[i].Data.Bid)); FCurrentData := TAskBid.LoadDataFile(FCurrentFileName);
inc(Idx); FCurrPosInFile := 0;
end; FCurrentIdx := 0;
FIsLiveData.SetValue(false);
FLastTimeStamp := 0;
FFilename := '';
FNextFileName := FindNextDataFile(FCurrentFileName);
if FNextFileName <> '' then
FNextData := TAskBid.LoadDataFile(FNextFileName);
exit;
end; end;
procedure TTABFileServer.Fetch; function TTABFileServer.GetChunk(var Data: array of TDataPoint<TAskBidItem>): Integer;
begin begin
Result := 0;
// Check if the source filename has changed.
if (FFilename <> FCurrentFileName) and (FFilename <> '') then
begin
// Filename has changed, initialize by loading the new file.
FCurrentFileName := FFilename;
FCurrentData := TAskBid.LoadDataFile(TLatch.Enqueue(FLoadGate), FCurrentFileName);
FCurrPosInFile := 0;
FCurrentIdx := 0;
FIsLiveData.SetValue(false);
FLastTimeStamp := 0;
FFilename := '';
FNextFileName := FindNextDataFile(FCurrentFileName);
if FNextFileName <> '' then
FNextData := TAskBid.LoadDataFile(TLatch.Enqueue(FLoadGate), FNextFileName);
end;
FHasNewData.Notify;
FNeedNewChunk := true;
FCurrChunk := nil;
end;
function TTABFileServer.GetData: TMutable<TArray<ITickData>>;
begin
Result := FData;
end;
function TTABFileServer.GetIsLiveData: TMutable<Boolean>;
begin
Result := FIsLiveData;
end;
function TTABFileServer.UpdateChunk: TArray<ITickData>;
begin
if not FNeedNewChunk then
exit(FCurrChunk);
FNeedNewChunk := false;
if FCurrentData.Done.IsSet then if FCurrentData.Done.IsSet then
begin begin
SetLength(FCurrChunk, FMaxFetch); var maxLen := Length(Data);
var n := 0;
while n < FMaxFetch do var currData := FCurrentData.Value;
while Result < MaxLen do
begin begin
if FCurrPosInFile >= Length(FCurrentData.Value) then if FCurrPosInFile >= Length(currData) then
begin begin
// Next File! // Next File!
FCurrentData := FNextData; FCurrentData := FNextData;
@@ -166,31 +118,38 @@ begin
begin begin
FNextFileName := FindNextDataFile(FCurrentFileName); FNextFileName := FindNextDataFile(FCurrentFileName);
if FNextFileName <> '' then if FNextFileName <> '' then
FNextData := TAskBid.LoadDataFile(TLatch.Enqueue(FLoadGate), FNextFileName); FNextData := TAskBid.LoadDataFile(FNextFileName);
if FCurrentData.Done.IsSet then
begin
currData := FCurrentData.Value;
continue;
end;
end end
else else
begin begin
// We're done with this series // We're done with this series
FIsLiveData.SetValue(true); FIsLiveData.SetValue(true);
end; end;
SetLength(FCurrChunk, n);
break; break;
end; end;
var currData := FCurrentData.Value[FCurrPosInFile]; var item := currData[FCurrPosInFile];
if FLastTimeStamp < currData.OADateTime then if FLastTimeStamp < item.TimeStamp then
begin begin
FLastTimeStamp := currData.OADateTime; FLastTimeStamp := item.TimeStamp;
FCurrChunk[n] := Data[Result].Create(FCurrentIdx, FLastTimeStamp, item.Data);
TDataPoint<ITick>.Create(FCurrentIdx, FLastTimeStamp, 1, TTick.Create(currData.Data.Ask, currData.Data.Bid)); inc(Result);
inc(FCurrentIdx); inc(FCurrentIdx);
inc(n);
end; end;
inc(FCurrPosInFile); inc(FCurrPosInFile);
end; end;
end; end;
Result := FCurrChunk; end;
function TTABFileServer.GetIsLiveData: TMutable<Boolean>;
begin
Result := FIsLiveData;
end; end;
end. end.